Skip to content

Commit

Permalink
SNOW-1271212 fixed unit test for bind uploader conversion
Browse files Browse the repository at this point in the history
  • Loading branch information
@sfc-gh-mhofman
sfc-gh-mhofman committed Apr 3, 2024
1 parent d3b151f commit 95fd416
Show file tree
Hide file tree
Showing 2 changed files with 95 additions and 53 deletions.
93 changes: 53 additions & 40 deletions Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -858,7 +858,7 @@ public void testExplicitDbTypeAssignmentForArrayValue()
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
/* Disabled until supported within automated tests environment available
/* TODO: Enable when features available on the automated tests environment
// HYBRID Tables
[TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.DATE, null, DbType.Date, FormatYmd, null)]
[TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.TIME, null, DbType.Time, FormatHms, null)]
Expand Down Expand Up @@ -893,7 +893,7 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
{
// Arrange
var timestamp = "2023/03/15 13:17:29.207 +05:00"; // 08:17:29.207 UTC
var expected = ExpectedTimestamp.From(timestamp, columnType);
var expected = ExpectedTimestampValueProvider.From(timestamp, columnType);
var columnWithPrecision = ColumnTypeWithPrecision(columnType, columnPrecision);
var testCase = $"TableType={tableType}, ColumnType={columnWithPrecision}, BindingType={bindingType}, ComparisonFormat={comparisonFormat}";
var bindingThreshold = 65280; // when exceeded enforces bindings via file on stage
Expand Down Expand Up @@ -924,14 +924,14 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
InsertMultipleRecords(conn, sqlInsert, bindingType, 2, expected, smallBatchRowCount, false);
InsertMultipleRecords(conn, sqlInsert, bindingType, smallBatchRowCount+2, expected, bigBatchRowCount, true);

// read all rows
// Assert
var row = 0;
using (var select = conn.CreateCommand($"select id, ts from {TableName} order by id"))
{
s_logger.Debug(select.CommandText);
var reader = select.ExecuteReader();
while (reader.Read())
{
// Assert
++row;
string faultMessage = $"Mismatch for row: {row}, {testCase}";
Assert.AreEqual(row, reader.GetInt32(0));
Expand All @@ -941,24 +941,38 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
Assert.AreEqual(1+smallBatchRowCount+bigBatchRowCount, row);
}

void InsertSingleRecord(IDbConnection conn, string sqlInsert, DbType binding, int identifier, ExpectedTimestamp ts)
void InsertSingleRecord(IDbConnection conn, string sqlInsert, DbType binding, int identifier, ExpectedTimestampValueProvider ts)
{
using (var insert = conn.CreateCommand(sqlInsert))
{
// Arrange
insert.Add("1", DbType.Int32, identifier);
insert.Add("2", binding, ts);
Assert.AreEqual(1, insert.ExecuteNonQuery());

// Act
s_logger.Info(sqlInsert);
var rowsAffected = insert.ExecuteNonQuery();

// Assert
Assert.AreEqual(1, rowsAffected);
Assert.IsNull(((SnowflakeDbCommand)insert).GetBindStage());
}
}

void InsertMultipleRecords(IDbConnection conn, string sqlInsert, DbType binding, int initialIdentifier, ExpectedTimestamp ts, int rowsCount, bool shouldUseBinding)
void InsertMultipleRecords(IDbConnection conn, string sqlInsert, DbType binding, int initialIdentifier, ExpectedTimestampValueProvider ts, int rowsCount, bool shouldUseBinding)
{
using (var insert = conn.CreateCommand(sqlInsert))
{
// Arrange
insert.Add("1", DbType.Int32, Enumerable.Range(initialIdentifier, rowsCount).ToArray());
insert.Add("2", binding, ts, rowsCount);
Assert.AreEqual(rowsCount, insert.ExecuteNonQuery());

// Act
s_logger.Debug(sqlInsert);
var rowsAffected = insert.ExecuteNonQuery();

// Assert
Assert.AreEqual(rowsCount, rowsAffected);
if (shouldUseBinding)
Assert.IsNotEmpty(((SnowflakeDbCommand)insert).GetBindStage());
else
Expand All @@ -971,60 +985,59 @@ private static string ColumnTypeWithPrecision(SFDataType columnType, Int32? colu
=> columnPrecision != null ? $"{columnType}({columnPrecision})" : $"{columnType}";
}

class ExpectedTimestamp
class ExpectedTimestampValueProvider
{
private readonly SFDataType _columnType;
private readonly DateTime? _expectedDateTime;
private readonly DateTimeOffset? _expectedDateTimeOffset;

internal static ExpectedTimestamp From(string timestampWithTimeZone, SFDataType columnType)
internal static ExpectedTimestampValueProvider From(string timestampWithTimeZone, SFDataType columnType)
{
if (columnType == SFDataType.TIMESTAMP_TZ)
return new ExpectedTimestamp(DateTimeOffset.ParseExact(timestampWithTimeZone,
"yyyy/MM/dd HH:mm:ss.fff zzz",
CultureInfo.InvariantCulture), columnType);

if (IsOffsetType(columnType))
return new ExpectedTimestamp(DateTimeOffset.ParseExact(timestampWithTimeZone,
return new ExpectedTimestampValueProvider(DateTimeOffset.ParseExact(timestampWithTimeZone,
"yyyy/MM/dd HH:mm:ss.fff zzz",
CultureInfo.InvariantCulture)
.ToLocalTime(), columnType);
CultureInfo.InvariantCulture), columnType);

return new ExpectedTimestamp(DateTime.ParseExact(timestampWithTimeZone,
return new ExpectedTimestampValueProvider(DateTime.ParseExact(timestampWithTimeZone,
"yyyy/MM/dd HH:mm:ss.fff zzz",
CultureInfo.InvariantCulture)
.ToLocalTime(), columnType);
CultureInfo.InvariantCulture), columnType);
}

private ExpectedTimestamp(DateTime dateTime, SFDataType columnType)
private ExpectedTimestampValueProvider(DateTime dateTime, SFDataType columnType)
{
_expectedDateTime = dateTime;
_expectedDateTimeOffset = null;
_columnType = columnType;
}

private ExpectedTimestamp(DateTimeOffset dateTimeOffset, SFDataType columnType)
private ExpectedTimestampValueProvider(DateTimeOffset dateTimeOffset, SFDataType columnType)
{
_expectedDateTimeOffset = dateTimeOffset;
_expectedDateTime = null;
_columnType = columnType;
}

internal SFDataType ExpectedColumnType() => _columnType;

internal void IsEqual(object actual, string comparisonFormat, string faultMessage)
{
if (IsOffsetType(_columnType))
{
Assert.AreEqual(GetDateTimeOffset().ToString(comparisonFormat),
((DateTimeOffset)actual).ToString(comparisonFormat),
faultMessage);
}
else
switch (_columnType)
{
Assert.AreEqual(GetDateTime().ToString(comparisonFormat),
((DateTime)actual).ToString(comparisonFormat),
faultMessage);
case SFDataType.TIMESTAMP_TZ:
Assert.AreEqual(GetDateTimeOffset().ToString(comparisonFormat),
((DateTimeOffset)actual).ToString(comparisonFormat),
faultMessage);
break;
case SFDataType.TIMESTAMP_LTZ:
Assert.AreEqual(GetDateTimeOffset().ToUniversalTime().ToString(comparisonFormat),
((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat),
faultMessage);
break;
default:
Assert.AreEqual(GetDateTime().ToString(comparisonFormat),
((DateTime)actual).ToString(comparisonFormat),
faultMessage);
break;
}
}

Expand Down Expand Up @@ -1053,15 +1066,15 @@ internal static string TableCreationFlags(this SFTableType val)
var externalVolume = Environment.GetEnvironmentVariable("ICEBERG_EXTERNAL_VOLUME");
var catalog = Environment.GetEnvironmentVariable("ICEBERG_CATALOG");
var baseLocation = Environment.GetEnvironmentVariable("ICEBERG_BASE_LOCATION");
Assert.IsNotNull(externalVolume);
Assert.IsNotNull(catalog);
Assert.IsNotNull(baseLocation);
Assert.IsNotNull(externalVolume, "env ICEBERG_EXTERNAL_VOLUME not set!");
Assert.IsNotNull(catalog, "env ICEBERG_CATALOG not set!");
Assert.IsNotNull(baseLocation, "env ICEBERG_BASE_LOCATION not set!");
return $"EXTERNAL_VOLUME = '{externalVolume}' CATALOG = '{catalog}' BASE_LOCATION = '{baseLocation}'";
}

internal static SnowflakeDbParameter Add(this IDbCommand command, string name, DbType dbType, object value)
{
if (value is ExpectedTimestamp expected)
if (value is ExpectedTimestampValueProvider expected)
return command.Add(name, dbType, expected);
var parameter = (SnowflakeDbParameter)command.CreateParameter();
parameter.ParameterName = name;
Expand All @@ -1071,7 +1084,7 @@ internal static SnowflakeDbParameter Add(this IDbCommand command, string name, D
return parameter;
}

internal static SnowflakeDbParameter Add(this IDbCommand command, string name, DbType dbType, ExpectedTimestamp value)
internal static SnowflakeDbParameter Add(this IDbCommand command, string name, DbType dbType, ExpectedTimestampValueProvider value)
{
if (dbType == DbType.DateTimeOffset)
{
Expand All @@ -1083,7 +1096,7 @@ internal static SnowflakeDbParameter Add(this IDbCommand command, string name, D
return command.Add(name, dbType, value.GetDateTime());
}

internal static void Add(this IDbCommand command, string name, DbType dbType, ExpectedTimestamp value, int batchSize)
internal static void Add(this IDbCommand command, string name, DbType dbType, ExpectedTimestampValueProvider value, int batchSize)
{
if (dbType == DbType.DateTimeOffset)
{
Expand Down
55 changes: 42 additions & 13 deletions Snowflake.Data.Tests/UnitTests/SFBindUploaderTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,17 @@ class SFBindUploaderTest
private readonly SFBindUploader _bindUploader = new SFBindUploader(null, "test");

[TestCase(SFDataType.DATE, "0", "1/1/1970")]
[TestCase(SFDataType.DATE, "73814400000", "5/4/1972")]
[TestCase(SFDataType.DATE, "1709193600000", "2/29/2024")]
[TestCase(SFDataType.DATE, "73785600000", "5/4/1972")]
[TestCase(SFDataType.DATE, "1709164800000", "2/29/2024")]
public void TestCsvDataConversionForDate(SFDataType dbType, string input, string expected)
{
DateTime dateExpected = DateTime.Parse(expected);
// Arrange
var dateExpected = DateTime.Parse(expected);
var check = SFDataConverter.csharpValToSfVal(SFDataType.DATE, dateExpected);
Assert.AreEqual(check, input);
// Act
DateTime dateActual = DateTime.Parse(_bindUploader.GetCSVData(dbType.ToString(), input));
// Assert
Assert.AreEqual(dateExpected, dateActual);
}

Expand All @@ -30,36 +35,57 @@ public void TestCsvDataConversionForDate(SFDataType dbType, string input, string
[TestCase(SFDataType.TIME, "46801000000000", "13:00:01.000000")]
public void TestCsvDataConversionForTime(SFDataType dbType, string input, string expected)
{
// Arrange
DateTime timeExpected = DateTime.Parse(expected);
var check = SFDataConverter.csharpValToSfVal(SFDataType.TIME, timeExpected);
Assert.AreEqual(check, input);
// Act
DateTime timeActual = DateTime.Parse(_bindUploader.GetCSVData(dbType.ToString(), input));
// Assert
Assert.AreEqual(timeExpected, timeActual);
}

[TestCase(SFDataType.TIMESTAMP_LTZ, "0", "2012-07-01T12:00:00.0000000+02:00")]
[TestCase(SFDataType.TIMESTAMP_LTZ, "1341144000000000000", "2012-07-01T12:00:00.0000000+02:00")]
[TestCase(SFDataType.TIMESTAMP_LTZ, "39600000000000", "1970-01-01T12:00:00.0000000+01:00")]
[TestCase(SFDataType.TIMESTAMP_LTZ, "1341136800000000000", "2012-07-01T12:00:00.0000000+02:00")]
[TestCase(SFDataType.TIMESTAMP_LTZ, "352245599987654000", "1981-02-28T23:59:59.9876540+02:00")]
[TestCase(SFDataType.TIMESTAMP_LTZ, "1678868249207000000", "2023/03/15T13:17:29.207+05:00")]
public void TestCsvDataConversionForTimestampLtz(SFDataType dbType, string input, string expected)
{
DateTimeOffset timestampExpected = DateTimeOffset.Parse(expected);
DateTimeOffset timestampActual = DateTimeOffset.Parse(_bindUploader.GetCSVData(dbType.ToString(), input));
Assert.AreEqual(timestampExpected, timestampActual);
// Arrange
var timestampExpected = DateTimeOffset.Parse(expected);
var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_LTZ, timestampExpected);
Assert.AreEqual(check, input);
// Act
var timestampActual = DateTimeOffset.Parse(_bindUploader.GetCSVData(dbType.ToString(), input));
// Assert
Assert.AreEqual(timestampExpected.ToLocalTime(), timestampActual);
}

[TestCase(SFDataType.TIMESTAMP_TZ, "1341136800000000000 1560", "2012-07-01 12:00:00.000000 +02:00")]
[TestCase(SFDataType.TIMESTAMP_TZ, "352245599987654000 1560", "1981-02-28 23:59:59.987654 +02:00")]
public void TestCsvDataConversionForTimestampTz(SFDataType dbType, string input, string expected)
{
// Arrange
DateTimeOffset timestampExpected = DateTimeOffset.Parse(expected);
var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_TZ, timestampExpected);
Assert.AreEqual(check, input);
// Act
DateTimeOffset timestampActual = DateTimeOffset.Parse(_bindUploader.GetCSVData(dbType.ToString(), input));
// Assert
Assert.AreEqual(timestampExpected, timestampActual);
}

[TestCase(SFDataType.TIMESTAMP_NTZ, "", "2012-07-01 12:00:00.000000 +02:00")]
[TestCase(SFDataType.TIMESTAMP_NTZ, "", "1981-02-28 23:59:59.987654 +02:00")]
[TestCase(SFDataType.TIMESTAMP_NTZ, "1341144000000000000", "2012-07-01 12:00:00.000000")]
[TestCase(SFDataType.TIMESTAMP_NTZ, "352252799987654000", "1981-02-28 23:59:59.987654")]
public void TestCsvDataConversionForTimestampNtz(SFDataType dbType, string input, string expected)
{
DateTimeOffset timestampExpected = DateTimeOffset.Parse(expected);
DateTimeOffset timestampActual = DateTimeOffset.Parse(_bindUploader.GetCSVData(dbType.ToString(), input));
// Arrange
DateTime timestampExpected = DateTime.Parse(expected);
var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_NTZ, timestampExpected);
Assert.AreEqual(check, input);
// Act
DateTime timestampActual = DateTime.Parse(_bindUploader.GetCSVData(dbType.ToString(), input));
// Assert
Assert.AreEqual(timestampExpected, timestampActual);
}

Expand All @@ -71,7 +97,10 @@ public void TestCsvDataConversionForTimestampNtz(SFDataType dbType, string input
[TestCase(SFDataType.TEXT, "Sample text", "Sample text")]
public void TestCsvDataConversionForText(SFDataType dbType, string input, string expected)
{
Assert.AreEqual(expected, _bindUploader.GetCSVData(dbType.ToString(), input));
// Act
var actual = _bindUploader.GetCSVData(dbType.ToString(), input);
// Assert
Assert.AreEqual(expected, actual);
}

}
Expand Down

0 comments on commit 95fd416

Please sign in to comment.