Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SNOW-1729244 support for large and small timestamps #1038

Merged
merged 9 commits into from
Oct 21, 2024
130 changes: 93 additions & 37 deletions Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved.
*/
#nullable enable
sfc-gh-knozderko marked this conversation as resolved.
Show resolved Hide resolved

using System;
using System.Data;
Expand Down Expand Up @@ -87,7 +88,7 @@ public void TestBindNullValue()
foreach (DbType type in Enum.GetValues(typeof(DbType)))
{
bool isTypeSupported = true;
string colName = null;
string colName;
using (IDbCommand command = dbConnection.CreateCommand())
{
var param = command.CreateParameter();
Expand Down Expand Up @@ -226,7 +227,7 @@ public void TestBindValue()
foreach (DbType type in Enum.GetValues(typeof(DbType)))
{
bool isTypeSupported = true;
string colName = null;
string colName;
using (IDbCommand command = dbConnection.CreateCommand())
{
var param = command.CreateParameter();
Expand Down Expand Up @@ -885,13 +886,20 @@ public void TestExplicitDbTypeAssignmentForArrayValue()
[TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
*/
// Session TimeZone cases
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")]
public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType, SFDataType columnType, Int32? columnPrecision, DbType bindingType, string comparisonFormat, string timeZone)
{
// Arrange
var timestamp = "2023/03/15 13:17:29.207 +05:00"; // 08:17:29.207 UTC
var expected = ExpectedTimestampWrapper.From(timestamp, columnType);
string[] timestamps =
{
"2023/03/15 13:17:29.207 +05:00",
"9999/12/30 23:24:25.987 +07:00",
"0001/01/02 02:06:07.000 -04:00"
};
var expected = ExpectedTimestampWrapper.From(timestamps, columnType);
var columnWithPrecision = ColumnTypeWithPrecision(columnType, columnPrecision);
var testCase = $"ResultFormat={resultFormat}, TableType={tableType}, ColumnType={columnWithPrecision}, BindingType={bindingType}, ComparisonFormat={comparisonFormat}";
var bindingThreshold = 65280; // when exceeded enforces bindings via file on stage
Expand All @@ -907,24 +915,34 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
if (!timeZone.IsNullOrEmpty()) // Driver ignores this setting and relies on local environment timezone
conn.ExecuteNonQuery($"alter session set TIMEZONE = '{timeZone}'");

// prepare initial column
var columns = new List<String> { "id number(10,0) not null primary key" };
var sql_columns = "id";
var sql_values = "?";

// prepare additional columns
for (int i = 1; i <= timestamps.Length; ++i)
{
columns.Add($"ts_{i} {columnWithPrecision}");
sql_columns += $",ts_{i}";
sql_values += ",?";
}

CreateOrReplaceTable(conn,
TableName,
tableType.TableDDLCreationPrefix(),
new[] {
"id number(10,0) not null primary key", // necessary only for HYBRID tables
$"ts {columnWithPrecision}"
},
columns,
tableType.TableDDLCreationFlags());

// Act+Assert
var sqlInsert = $"insert into {TableName} (id, ts) values (?, ?)";
var sqlInsert = $"insert into {TableName} ({sql_columns}) values ({sql_values})";
InsertSingleRecord(conn, sqlInsert, bindingType, 1, expected);
InsertMultipleRecords(conn, sqlInsert, bindingType, 2, expected, smallBatchRowCount, false);
InsertMultipleRecords(conn, sqlInsert, bindingType, smallBatchRowCount+2, expected, bigBatchRowCount, true);

// Assert
var row = 0;
using (var select = conn.CreateCommand($"select id, ts from {TableName} order by id"))
using (var select = conn.CreateCommand($"select {sql_columns} from {TableName} order by id"))
{
s_logger.Debug(select.CommandText);
var reader = select.ExecuteReader();
Expand All @@ -933,7 +951,11 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
++row;
string faultMessage = $"Mismatch for row: {row}, {testCase}";
Assert.AreEqual(row, reader.GetInt32(0));
expected.AssertEqual(reader.GetValue(1), comparisonFormat, faultMessage);

for (int i = 0; i < timestamps.Length; ++i)
{
expected.AssertEqual(reader.GetValue(i + 1), comparisonFormat, faultMessage, i);
}
}
}
Assert.AreEqual(1+smallBatchRowCount+bigBatchRowCount, row);
Expand All @@ -948,12 +970,24 @@ private void InsertSingleRecord(IDbConnection conn, string sqlInsert, DbType bin
insert.Add("1", DbType.Int32, identifier);
if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType()))
{
var parameter = (SnowflakeDbParameter)insert.Add("2", binding, ts.GetDateTimeOffset());
parameter.SFDataType = ts.ExpectedColumnType();
var dateTimeOffsets = ts.GetDateTimeOffsets();
for (int i = 0; i < dateTimeOffsets.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = dateTimeOffsets[i];
var parameter = insert.Add(parameterName, binding, parameterValue);
parameter.SFDataType = ts.ExpectedColumnType();
}
}
else
{
insert.Add("2", binding, ts.GetDateTime());
var dateTimes = ts.GetDateTimes();
for (int i = 0; i < dateTimes.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = dateTimes[i];
insert.Add(parameterName, binding, parameterValue);
}
}

// Act
Expand All @@ -974,12 +1008,25 @@ private void InsertMultipleRecords(IDbConnection conn, string sqlInsert, DbType
insert.Add("1", DbType.Int32, Enumerable.Range(initialIdentifier, rowsCount).ToArray());
if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType()))
{
var parameter = (SnowflakeDbParameter)insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTimeOffset(), rowsCount).ToArray());
parameter.SFDataType = ts.ExpectedColumnType();
var dateTimeOffsets = ts.GetDateTimeOffsets();
for (int i = 0; i < dateTimeOffsets.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = Enumerable.Repeat(dateTimeOffsets[i], rowsCount).ToArray();
var parameter = insert.Add(parameterName, binding, parameterValue);
parameter.SFDataType = ts.ExpectedColumnType();
}

}
else
{
insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTime(), rowsCount).ToArray());
var dateTimes = ts.GetDateTimes();
for (int i = 0; i < dateTimes.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = Enumerable.Repeat(dateTimes[i], rowsCount).ToArray();
insert.Add(parameterName, binding, parameterValue);
}
}

// Act
Expand All @@ -1002,57 +1049,66 @@ private static string ColumnTypeWithPrecision(SFDataType columnType, Int32? colu
class ExpectedTimestampWrapper
{
private readonly SFDataType _columnType;
private readonly DateTime? _expectedDateTime;
private readonly DateTimeOffset? _expectedDateTimeOffset;
private readonly DateTime[]? _expectedDateTimes;
private readonly DateTimeOffset[]? _expectedDateTimeOffsets;

internal static ExpectedTimestampWrapper From(string timestampWithTimeZone, SFDataType columnType)
internal static ExpectedTimestampWrapper From(string[] timestampsWithTimeZone, SFDataType columnType)
{
if (IsOffsetType(columnType))
{
var dateTimeOffset = DateTimeOffset.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture);
return new ExpectedTimestampWrapper(dateTimeOffset, columnType);
var dateTimeOffsets =
timestampsWithTimeZone
.Select(ts => DateTimeOffset.ParseExact(ts, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture))
.ToArray();
return new ExpectedTimestampWrapper(dateTimeOffsets, columnType);
}

var dateTime = DateTime.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture);
return new ExpectedTimestampWrapper(dateTime, columnType);
var dateTimes =
timestampsWithTimeZone
.Select(ts => DateTime.ParseExact(ts, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture))
.ToArray();

return new ExpectedTimestampWrapper(dateTimes, columnType);
}

private ExpectedTimestampWrapper(DateTime dateTime, SFDataType columnType)
private ExpectedTimestampWrapper(DateTime[] dateTimes, SFDataType columnType)
{
_expectedDateTime = dateTime;
_expectedDateTimeOffset = null;
_expectedDateTimes = dateTimes;
_expectedDateTimeOffsets = null;
_columnType = columnType;
}

private ExpectedTimestampWrapper(DateTimeOffset dateTimeOffset, SFDataType columnType)
private ExpectedTimestampWrapper(DateTimeOffset[] dateTimeOffsets, SFDataType columnType)
{
_expectedDateTimeOffset = dateTimeOffset;
_expectedDateTime = null;
_expectedDateTimeOffsets = dateTimeOffsets;
_expectedDateTimes = null;
_columnType = columnType;
}

internal SFDataType ExpectedColumnType() => _columnType;

internal void AssertEqual(object actual, string comparisonFormat, string faultMessage)
internal void AssertEqual(object actual, string comparisonFormat, string faultMessage, int index)
{
switch (_columnType)
{
case SFDataType.TIMESTAMP_TZ:
Assert.AreEqual(GetDateTimeOffset().ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage);
Assert.AreEqual(GetDateTimeOffsets()[index].ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage);
break;
case SFDataType.TIMESTAMP_LTZ:
Assert.AreEqual(GetDateTimeOffset().ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage);
Assert.AreEqual(GetDateTimeOffsets()[index].ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage);
break;
default:
Assert.AreEqual(GetDateTime().ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage);
Assert.AreEqual(GetDateTimes()[index].ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage);
break;
}
}

internal DateTime GetDateTime() => _expectedDateTime ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");
internal DateTime[] GetDateTimes() => _expectedDateTimes ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");

internal DateTimeOffset GetDateTimeOffset() => _expectedDateTimeOffset ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");
internal DateTimeOffset[] GetDateTimeOffsets() => _expectedDateTimeOffsets ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");

internal static bool IsOffsetType(SFDataType type) => type == SFDataType.TIMESTAMP_LTZ || type == SFDataType.TIMESTAMP_TZ;
}
}

#nullable restore
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,41 @@ internal static IEnumerable<object[]> DateTimeConversionCases()
null,
DateTime.Parse("2024-07-11 21:20:05.1234568").ToLocalTime()
};
yield return new object[]
{
"9999-12-31 23:59:59.999999",
SFDataType.TIMESTAMP_NTZ.ToString(),
DateTime.Parse("9999-12-31 23:59:59.999999"),
DateTime.Parse("9999-12-31 23:59:59.999999")
};
yield return new object[]
{
"9999-12-31 23:59:59.999999 +1:00",
SFDataType.TIMESTAMP_TZ.ToString(),
null,
DateTime.SpecifyKind(DateTime.Parse("9999-12-31 22:59:59.999999"), DateTimeKind.Utc)
};
yield return new object[]
{
"9999-12-31 23:59:59.999999 +13:00",
SFDataType.TIMESTAMP_LTZ.ToString(),
sfc-gh-dstempniak marked this conversation as resolved.
Show resolved Hide resolved
null,
DateTime.Parse("9999-12-31 10:59:59.999999").ToLocalTime()
};
yield return new object[]
{
"0001-01-01 00:00:00",
SFDataType.TIMESTAMP_NTZ.ToString(),
DateTime.Parse("0001-01-01 00:00:00"),
DateTime.Parse("0001-01-01 00:00:00")
};
yield return new object[]
{
"0001-01-01 00:00:00 -1:00",
SFDataType.TIMESTAMP_TZ.ToString(),
null,
DateTime.SpecifyKind(DateTime.Parse("0001-01-01 01:00:00"), DateTimeKind.Utc)
};
}

[Test]
Expand Down Expand Up @@ -445,6 +480,41 @@ internal static IEnumerable<object[]> DateTimeOffsetConversionCases()
null,
DateTimeOffset.Parse("2024-07-11 14:20:05.1234568 -7:00")
};
yield return new object[]
{
"9999-12-31 23:59:59.999999",
SFDataType.TIMESTAMP_NTZ.ToString(),
DateTime.Parse("9999-12-31 23:59:59.999999"),
DateTimeOffset.Parse("9999-12-31 23:59:59.999999Z")
};
yield return new object[]
{
"9999-12-31 23:59:59.999999 +1:00",
SFDataType.TIMESTAMP_TZ.ToString(),
null,
DateTimeOffset.Parse("9999-12-31 23:59:59.999999 +1:00")
};
yield return new object[]
{
"9999-12-31 23:59:59.999999 +13:00",
SFDataType.TIMESTAMP_LTZ.ToString(),
sfc-gh-dstempniak marked this conversation as resolved.
Show resolved Hide resolved
null,
DateTimeOffset.Parse("9999-12-31 23:59:59.999999 +13:00")
};
yield return new object[]
{
"0001-01-01 00:00:00",
SFDataType.TIMESTAMP_NTZ.ToString(),
DateTime.Parse("0001-01-01 00:00:00"),
DateTimeOffset.Parse("0001-01-01 00:00:00Z")
};
yield return new object[]
{
"0001-01-01 00:00:00 -1:00",
SFDataType.TIMESTAMP_TZ.ToString(),
null,
DateTimeOffset.Parse("0001-01-01 00:00:00 -1:00")
};
}

private TimeZoneInfo GetTimeZone(SnowflakeDbConnection connection)
Expand Down
Loading
Loading