Skip to content

Commit

Permalink
SNOW-1729244 support for large and small timestamps (#1038)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sfc-gh-dstempniak
sfc-gh-dstempniak authored Oct 21, 2024
1 parent b146410 commit 90b7007
Show file tree
Hide file tree
Showing 10 changed files with 271 additions and 88 deletions.
130 changes: 93 additions & 37 deletions Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved.
*/
#nullable enable

using System;
using System.Data;
Expand Down Expand Up @@ -87,7 +88,7 @@ public void TestBindNullValue()
foreach (DbType type in Enum.GetValues(typeof(DbType)))
{
bool isTypeSupported = true;
string colName = null;
string colName;
using (IDbCommand command = dbConnection.CreateCommand())
{
var param = command.CreateParameter();
Expand Down Expand Up @@ -226,7 +227,7 @@ public void TestBindValue()
foreach (DbType type in Enum.GetValues(typeof(DbType)))
{
bool isTypeSupported = true;
string colName = null;
string colName;
using (IDbCommand command = dbConnection.CreateCommand())
{
var param = command.CreateParameter();
Expand Down Expand Up @@ -885,13 +886,20 @@ public void TestExplicitDbTypeAssignmentForArrayValue()
[TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
*/
// Session TimeZone cases
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")]
public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType, SFDataType columnType, Int32? columnPrecision, DbType bindingType, string comparisonFormat, string timeZone)
{
// Arrange
var timestamp = "2023/03/15 13:17:29.207 +05:00"; // 08:17:29.207 UTC
var expected = ExpectedTimestampWrapper.From(timestamp, columnType);
string[] timestamps =
{
"2023/03/15 13:17:29.207 +05:00",
"9999/12/30 23:24:25.987 +07:00",
"0001/01/02 02:06:07.000 -04:00"
};
var expected = ExpectedTimestampWrapper.From(timestamps, columnType);
var columnWithPrecision = ColumnTypeWithPrecision(columnType, columnPrecision);
var testCase = $"ResultFormat={resultFormat}, TableType={tableType}, ColumnType={columnWithPrecision}, BindingType={bindingType}, ComparisonFormat={comparisonFormat}";
var bindingThreshold = 65280; // when exceeded enforces bindings via file on stage
Expand All @@ -907,24 +915,34 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
if (!timeZone.IsNullOrEmpty()) // Driver ignores this setting and relies on local environment timezone
conn.ExecuteNonQuery($"alter session set TIMEZONE = '{timeZone}'");

// prepare initial column
var columns = new List<String> { "id number(10,0) not null primary key" };
var sql_columns = "id";
var sql_values = "?";

// prepare additional columns
for (int i = 1; i <= timestamps.Length; ++i)
{
columns.Add($"ts_{i} {columnWithPrecision}");
sql_columns += $",ts_{i}";
sql_values += ",?";
}

CreateOrReplaceTable(conn,
TableName,
tableType.TableDDLCreationPrefix(),
new[] {
"id number(10,0) not null primary key", // necessary only for HYBRID tables
$"ts {columnWithPrecision}"
},
columns,
tableType.TableDDLCreationFlags());

// Act+Assert
var sqlInsert = $"insert into {TableName} (id, ts) values (?, ?)";
var sqlInsert = $"insert into {TableName} ({sql_columns}) values ({sql_values})";
InsertSingleRecord(conn, sqlInsert, bindingType, 1, expected);
InsertMultipleRecords(conn, sqlInsert, bindingType, 2, expected, smallBatchRowCount, false);
InsertMultipleRecords(conn, sqlInsert, bindingType, smallBatchRowCount+2, expected, bigBatchRowCount, true);

// Assert
var row = 0;
using (var select = conn.CreateCommand($"select id, ts from {TableName} order by id"))
using (var select = conn.CreateCommand($"select {sql_columns} from {TableName} order by id"))
{
s_logger.Debug(select.CommandText);
var reader = select.ExecuteReader();
Expand All @@ -933,7 +951,11 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
++row;
string faultMessage = $"Mismatch for row: {row}, {testCase}";
Assert.AreEqual(row, reader.GetInt32(0));
expected.AssertEqual(reader.GetValue(1), comparisonFormat, faultMessage);

for (int i = 0; i < timestamps.Length; ++i)
{
expected.AssertEqual(reader.GetValue(i + 1), comparisonFormat, faultMessage, i);
}
}
}
Assert.AreEqual(1+smallBatchRowCount+bigBatchRowCount, row);
Expand All @@ -948,12 +970,24 @@ private void InsertSingleRecord(IDbConnection conn, string sqlInsert, DbType bin
insert.Add("1", DbType.Int32, identifier);
if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType()))
{
var parameter = (SnowflakeDbParameter)insert.Add("2", binding, ts.GetDateTimeOffset());
parameter.SFDataType = ts.ExpectedColumnType();
var dateTimeOffsets = ts.GetDateTimeOffsets();
for (int i = 0; i < dateTimeOffsets.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = dateTimeOffsets[i];
var parameter = insert.Add(parameterName, binding, parameterValue);
parameter.SFDataType = ts.ExpectedColumnType();
}
}
else
{
insert.Add("2", binding, ts.GetDateTime());
var dateTimes = ts.GetDateTimes();
for (int i = 0; i < dateTimes.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = dateTimes[i];
insert.Add(parameterName, binding, parameterValue);
}
}

// Act
Expand All @@ -974,12 +1008,25 @@ private void InsertMultipleRecords(IDbConnection conn, string sqlInsert, DbType
insert.Add("1", DbType.Int32, Enumerable.Range(initialIdentifier, rowsCount).ToArray());
if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType()))
{
var parameter = (SnowflakeDbParameter)insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTimeOffset(), rowsCount).ToArray());
parameter.SFDataType = ts.ExpectedColumnType();
var dateTimeOffsets = ts.GetDateTimeOffsets();
for (int i = 0; i < dateTimeOffsets.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = Enumerable.Repeat(dateTimeOffsets[i], rowsCount).ToArray();
var parameter = insert.Add(parameterName, binding, parameterValue);
parameter.SFDataType = ts.ExpectedColumnType();
}

}
else
{
insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTime(), rowsCount).ToArray());
var dateTimes = ts.GetDateTimes();
for (int i = 0; i < dateTimes.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = Enumerable.Repeat(dateTimes[i], rowsCount).ToArray();
insert.Add(parameterName, binding, parameterValue);
}
}

// Act
Expand All @@ -1002,57 +1049,66 @@ private static string ColumnTypeWithPrecision(SFDataType columnType, Int32? colu
class ExpectedTimestampWrapper
{
private readonly SFDataType _columnType;
private readonly DateTime? _expectedDateTime;
private readonly DateTimeOffset? _expectedDateTimeOffset;
private readonly DateTime[]? _expectedDateTimes;
private readonly DateTimeOffset[]? _expectedDateTimeOffsets;

internal static ExpectedTimestampWrapper From(string timestampWithTimeZone, SFDataType columnType)
internal static ExpectedTimestampWrapper From(string[] timestampsWithTimeZone, SFDataType columnType)
{
if (IsOffsetType(columnType))
{
var dateTimeOffset = DateTimeOffset.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture);
return new ExpectedTimestampWrapper(dateTimeOffset, columnType);
var dateTimeOffsets =
timestampsWithTimeZone
.Select(ts => DateTimeOffset.ParseExact(ts, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture))
.ToArray();
return new ExpectedTimestampWrapper(dateTimeOffsets, columnType);
}

var dateTime = DateTime.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture);
return new ExpectedTimestampWrapper(dateTime, columnType);
var dateTimes =
timestampsWithTimeZone
.Select(ts => DateTime.ParseExact(ts, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture))
.ToArray();

return new ExpectedTimestampWrapper(dateTimes, columnType);
}

private ExpectedTimestampWrapper(DateTime dateTime, SFDataType columnType)
private ExpectedTimestampWrapper(DateTime[] dateTimes, SFDataType columnType)
{
_expectedDateTime = dateTime;
_expectedDateTimeOffset = null;
_expectedDateTimes = dateTimes;
_expectedDateTimeOffsets = null;
_columnType = columnType;
}

private ExpectedTimestampWrapper(DateTimeOffset dateTimeOffset, SFDataType columnType)
private ExpectedTimestampWrapper(DateTimeOffset[] dateTimeOffsets, SFDataType columnType)
{
_expectedDateTimeOffset = dateTimeOffset;
_expectedDateTime = null;
_expectedDateTimeOffsets = dateTimeOffsets;
_expectedDateTimes = null;
_columnType = columnType;
}

internal SFDataType ExpectedColumnType() => _columnType;

internal void AssertEqual(object actual, string comparisonFormat, string faultMessage)
internal void AssertEqual(object actual, string comparisonFormat, string faultMessage, int index)
{
switch (_columnType)
{
case SFDataType.TIMESTAMP_TZ:
Assert.AreEqual(GetDateTimeOffset().ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage);
Assert.AreEqual(GetDateTimeOffsets()[index].ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage);
break;
case SFDataType.TIMESTAMP_LTZ:
Assert.AreEqual(GetDateTimeOffset().ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage);
Assert.AreEqual(GetDateTimeOffsets()[index].ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage);
break;
default:
Assert.AreEqual(GetDateTime().ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage);
Assert.AreEqual(GetDateTimes()[index].ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage);
break;
}
}

internal DateTime GetDateTime() => _expectedDateTime ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");
internal DateTime[] GetDateTimes() => _expectedDateTimes ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");

internal DateTimeOffset GetDateTimeOffset() => _expectedDateTimeOffset ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");
internal DateTimeOffset[] GetDateTimeOffsets() => _expectedDateTimeOffsets ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");

internal static bool IsOffsetType(SFDataType type) => type == SFDataType.TIMESTAMP_LTZ || type == SFDataType.TIMESTAMP_TZ;
}
}

#nullable restore
70 changes: 70 additions & 0 deletions Snowflake.Data.Tests/IntegrationTests/StructuredTypesWithEmbeddedUnstructuredIT.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,41 @@ internal static IEnumerable<object[]> DateTimeConversionCases()
null,
DateTime.Parse("2024-07-11 21:20:05.1234568").ToLocalTime()
};
yield return new object[]
{
"9999-12-31 23:59:59.999999",
SFDataType.TIMESTAMP_NTZ.ToString(),
DateTime.Parse("9999-12-31 23:59:59.999999"),
DateTime.Parse("9999-12-31 23:59:59.999999")
};
yield return new object[]
{
"9999-12-31 23:59:59.999999 +1:00",
SFDataType.TIMESTAMP_TZ.ToString(),
null,
DateTime.SpecifyKind(DateTime.Parse("9999-12-31 22:59:59.999999"), DateTimeKind.Utc)
};
yield return new object[]
{
"9999-12-31 23:59:59.999999 +13:00",
SFDataType.TIMESTAMP_LTZ.ToString(),
null,
DateTime.Parse("9999-12-31 10:59:59.999999").ToLocalTime()
};
yield return new object[]
{
"0001-01-01 00:00:00",
SFDataType.TIMESTAMP_NTZ.ToString(),
DateTime.Parse("0001-01-01 00:00:00"),
DateTime.Parse("0001-01-01 00:00:00")
};
yield return new object[]
{
"0001-01-01 00:00:00 -1:00",
SFDataType.TIMESTAMP_TZ.ToString(),
null,
DateTime.SpecifyKind(DateTime.Parse("0001-01-01 01:00:00"), DateTimeKind.Utc)
};
}

[Test]
Expand Down Expand Up @@ -445,6 +480,41 @@ internal static IEnumerable<object[]> DateTimeOffsetConversionCases()
null,
DateTimeOffset.Parse("2024-07-11 14:20:05.1234568 -7:00")
};
yield return new object[]
{
"9999-12-31 23:59:59.999999",
SFDataType.TIMESTAMP_NTZ.ToString(),
DateTime.Parse("9999-12-31 23:59:59.999999"),
DateTimeOffset.Parse("9999-12-31 23:59:59.999999Z")
};
yield return new object[]
{
"9999-12-31 23:59:59.999999 +1:00",
SFDataType.TIMESTAMP_TZ.ToString(),
null,
DateTimeOffset.Parse("9999-12-31 23:59:59.999999 +1:00")
};
yield return new object[]
{
"9999-12-31 23:59:59.999999 +13:00",
SFDataType.TIMESTAMP_LTZ.ToString(),
null,
DateTimeOffset.Parse("9999-12-31 23:59:59.999999 +13:00")
};
yield return new object[]
{
"0001-01-01 00:00:00",
SFDataType.TIMESTAMP_NTZ.ToString(),
DateTime.Parse("0001-01-01 00:00:00"),
DateTimeOffset.Parse("0001-01-01 00:00:00Z")
};
yield return new object[]
{
"0001-01-01 00:00:00 -1:00",
SFDataType.TIMESTAMP_TZ.ToString(),
null,
DateTimeOffset.Parse("0001-01-01 00:00:00 -1:00")
};
}

private TimeZoneInfo GetTimeZone(SnowflakeDbConnection connection)
Expand Down
Loading

0 comments on commit 90b7007

Please sign in to comment.