From 90b700753dff8c6b2fb7031bc2ca8651e25aa722 Mon Sep 17 00:00:00 2001 From: Dariusz Stempniak Date: Mon, 21 Oct 2024 11:36:30 +0200 Subject: [PATCH] SNOW-1729244 support for large and small timestamps (#1038) --- .../IntegrationTests/SFBindTestIT.cs | 130 +++++++++++++----- ...ructuredTypesWithEmbeddedUnstructuredIT.cs | 70 ++++++++++ .../UnitTests/SFBindUploaderTest.cs | 29 ++-- .../UnitTests/SFDataConverterTest.cs | 30 +++- .../UnitTests/StructuredTypesTest.cs | 12 ++ Snowflake.Data/Client/SnowflakeDbCommand.cs | 8 +- Snowflake.Data/Core/ArrowResultSet.cs | 2 +- Snowflake.Data/Core/SFBindUploader.cs | 28 ++-- Snowflake.Data/Core/SFDataConverter.cs | 48 +++---- Snowflake.Data/Core/SFResultSet.cs | 2 +- 10 files changed, 271 insertions(+), 88 deletions(-) diff --git a/Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs index 956362fe8..05995e0d4 100755 --- a/Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs @@ -1,6 +1,7 @@ /* * Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. */ +#nullable enable using System; using System.Data; @@ -87,7 +88,7 @@ public void TestBindNullValue() foreach (DbType type in Enum.GetValues(typeof(DbType))) { bool isTypeSupported = true; - string colName = null; + string colName; using (IDbCommand command = dbConnection.CreateCommand()) { var param = command.CreateParameter(); @@ -226,7 +227,7 @@ public void TestBindValue() foreach (DbType type in Enum.GetValues(typeof(DbType))) { bool isTypeSupported = true; - string colName = null; + string colName; using (IDbCommand command = dbConnection.CreateCommand()) { var param = command.CreateParameter(); @@ -885,13 +886,20 @@ public void TestExplicitDbTypeAssignmentForArrayValue() [TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] */ // Session TimeZone cases - [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")] + [TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")] [TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")] + [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")] + [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")] public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType, SFDataType columnType, Int32? columnPrecision, DbType bindingType, string comparisonFormat, string timeZone) { // Arrange - var timestamp = "2023/03/15 13:17:29.207 +05:00"; // 08:17:29.207 UTC - var expected = ExpectedTimestampWrapper.From(timestamp, columnType); + string[] timestamps = + { + "2023/03/15 13:17:29.207 +05:00", + "9999/12/30 23:24:25.987 +07:00", + "0001/01/02 02:06:07.000 -04:00" + }; + var expected = ExpectedTimestampWrapper.From(timestamps, columnType); var columnWithPrecision = ColumnTypeWithPrecision(columnType, columnPrecision); var testCase = $"ResultFormat={resultFormat}, TableType={tableType}, ColumnType={columnWithPrecision}, BindingType={bindingType}, ComparisonFormat={comparisonFormat}"; var bindingThreshold = 65280; // when exceeded enforces bindings via file on stage @@ -907,24 +915,34 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType if (!timeZone.IsNullOrEmpty()) // Driver ignores this setting and relies on local environment timezone conn.ExecuteNonQuery($"alter session set TIMEZONE = '{timeZone}'"); + // prepare initial column + var columns = new List { "id number(10,0) not null primary key" }; + var sql_columns = "id"; + var sql_values = "?"; + + // prepare additional columns + for (int i = 1; i <= timestamps.Length; ++i) + { + columns.Add($"ts_{i} {columnWithPrecision}"); + sql_columns += $",ts_{i}"; + sql_values += ",?"; + } + CreateOrReplaceTable(conn, TableName, tableType.TableDDLCreationPrefix(), - new[] { - "id number(10,0) not null primary key", // necessary only for HYBRID tables - $"ts {columnWithPrecision}" - }, + columns, tableType.TableDDLCreationFlags()); // Act+Assert - var sqlInsert = $"insert into {TableName} (id, ts) values (?, ?)"; + var sqlInsert = $"insert into {TableName} ({sql_columns}) values ({sql_values})"; InsertSingleRecord(conn, sqlInsert, bindingType, 1, expected); InsertMultipleRecords(conn, sqlInsert, bindingType, 2, expected, smallBatchRowCount, false); InsertMultipleRecords(conn, sqlInsert, bindingType, smallBatchRowCount+2, expected, bigBatchRowCount, true); // Assert var row = 0; - using (var select = conn.CreateCommand($"select id, ts from {TableName} order by id")) + using (var select = conn.CreateCommand($"select {sql_columns} from {TableName} order by id")) { s_logger.Debug(select.CommandText); var reader = select.ExecuteReader(); @@ -933,7 +951,11 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType ++row; string faultMessage = $"Mismatch for row: {row}, {testCase}"; Assert.AreEqual(row, reader.GetInt32(0)); - expected.AssertEqual(reader.GetValue(1), comparisonFormat, faultMessage); + + for (int i = 0; i < timestamps.Length; ++i) + { + expected.AssertEqual(reader.GetValue(i + 1), comparisonFormat, faultMessage, i); + } } } Assert.AreEqual(1+smallBatchRowCount+bigBatchRowCount, row); @@ -948,12 +970,24 @@ private void InsertSingleRecord(IDbConnection conn, string sqlInsert, DbType bin insert.Add("1", DbType.Int32, identifier); if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType())) { - var parameter = (SnowflakeDbParameter)insert.Add("2", binding, ts.GetDateTimeOffset()); - parameter.SFDataType = ts.ExpectedColumnType(); + var dateTimeOffsets = ts.GetDateTimeOffsets(); + for (int i = 0; i < dateTimeOffsets.Length; ++i) + { + var parameterName = (i + 2).ToString(); + var parameterValue = dateTimeOffsets[i]; + var parameter = insert.Add(parameterName, binding, parameterValue); + parameter.SFDataType = ts.ExpectedColumnType(); + } } else { - insert.Add("2", binding, ts.GetDateTime()); + var dateTimes = ts.GetDateTimes(); + for (int i = 0; i < dateTimes.Length; ++i) + { + var parameterName = (i + 2).ToString(); + var parameterValue = dateTimes[i]; + insert.Add(parameterName, binding, parameterValue); + } } // Act @@ -974,12 +1008,25 @@ private void InsertMultipleRecords(IDbConnection conn, string sqlInsert, DbType insert.Add("1", DbType.Int32, Enumerable.Range(initialIdentifier, rowsCount).ToArray()); if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType())) { - var parameter = (SnowflakeDbParameter)insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTimeOffset(), rowsCount).ToArray()); - parameter.SFDataType = ts.ExpectedColumnType(); + var dateTimeOffsets = ts.GetDateTimeOffsets(); + for (int i = 0; i < dateTimeOffsets.Length; ++i) + { + var parameterName = (i + 2).ToString(); + var parameterValue = Enumerable.Repeat(dateTimeOffsets[i], rowsCount).ToArray(); + var parameter = insert.Add(parameterName, binding, parameterValue); + parameter.SFDataType = ts.ExpectedColumnType(); + } + } else { - insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTime(), rowsCount).ToArray()); + var dateTimes = ts.GetDateTimes(); + for (int i = 0; i < dateTimes.Length; ++i) + { + var parameterName = (i + 2).ToString(); + var parameterValue = Enumerable.Repeat(dateTimes[i], rowsCount).ToArray(); + insert.Add(parameterName, binding, parameterValue); + } } // Act @@ -1002,57 +1049,66 @@ private static string ColumnTypeWithPrecision(SFDataType columnType, Int32? colu class ExpectedTimestampWrapper { private readonly SFDataType _columnType; - private readonly DateTime? _expectedDateTime; - private readonly DateTimeOffset? _expectedDateTimeOffset; + private readonly DateTime[]? _expectedDateTimes; + private readonly DateTimeOffset[]? _expectedDateTimeOffsets; - internal static ExpectedTimestampWrapper From(string timestampWithTimeZone, SFDataType columnType) + internal static ExpectedTimestampWrapper From(string[] timestampsWithTimeZone, SFDataType columnType) { if (IsOffsetType(columnType)) { - var dateTimeOffset = DateTimeOffset.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture); - return new ExpectedTimestampWrapper(dateTimeOffset, columnType); + var dateTimeOffsets = + timestampsWithTimeZone + .Select(ts => DateTimeOffset.ParseExact(ts, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture)) + .ToArray(); + return new ExpectedTimestampWrapper(dateTimeOffsets, columnType); } - var dateTime = DateTime.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture); - return new ExpectedTimestampWrapper(dateTime, columnType); + var dateTimes = + timestampsWithTimeZone + .Select(ts => DateTime.ParseExact(ts, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture)) + .ToArray(); + + return new ExpectedTimestampWrapper(dateTimes, columnType); } - private ExpectedTimestampWrapper(DateTime dateTime, SFDataType columnType) + private ExpectedTimestampWrapper(DateTime[] dateTimes, SFDataType columnType) { - _expectedDateTime = dateTime; - _expectedDateTimeOffset = null; + _expectedDateTimes = dateTimes; + _expectedDateTimeOffsets = null; _columnType = columnType; } - private ExpectedTimestampWrapper(DateTimeOffset dateTimeOffset, SFDataType columnType) + private ExpectedTimestampWrapper(DateTimeOffset[] dateTimeOffsets, SFDataType columnType) { - _expectedDateTimeOffset = dateTimeOffset; - _expectedDateTime = null; + _expectedDateTimeOffsets = dateTimeOffsets; + _expectedDateTimes = null; _columnType = columnType; } internal SFDataType ExpectedColumnType() => _columnType; - internal void AssertEqual(object actual, string comparisonFormat, string faultMessage) + internal void AssertEqual(object actual, string comparisonFormat, string faultMessage, int index) { switch (_columnType) { case SFDataType.TIMESTAMP_TZ: - Assert.AreEqual(GetDateTimeOffset().ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage); + Assert.AreEqual(GetDateTimeOffsets()[index].ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage); break; case SFDataType.TIMESTAMP_LTZ: - Assert.AreEqual(GetDateTimeOffset().ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage); + Assert.AreEqual(GetDateTimeOffsets()[index].ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage); break; default: - Assert.AreEqual(GetDateTime().ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage); + Assert.AreEqual(GetDateTimes()[index].ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage); break; } } - internal DateTime GetDateTime() => _expectedDateTime ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}"); + internal DateTime[] GetDateTimes() => _expectedDateTimes ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}"); - internal DateTimeOffset GetDateTimeOffset() => _expectedDateTimeOffset ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}"); + internal DateTimeOffset[] GetDateTimeOffsets() => _expectedDateTimeOffsets ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}"); internal static bool IsOffsetType(SFDataType type) => type == SFDataType.TIMESTAMP_LTZ || type == SFDataType.TIMESTAMP_TZ; } } + +#nullable restore diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredTypesWithEmbeddedUnstructuredIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredTypesWithEmbeddedUnstructuredIT.cs index 784aa4132..22f8310a1 100644 --- a/Snowflake.Data.Tests/IntegrationTests/StructuredTypesWithEmbeddedUnstructuredIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredTypesWithEmbeddedUnstructuredIT.cs @@ -361,6 +361,41 @@ internal static IEnumerable DateTimeConversionCases() null, DateTime.Parse("2024-07-11 21:20:05.1234568").ToLocalTime() }; + yield return new object[] + { + "9999-12-31 23:59:59.999999", + SFDataType.TIMESTAMP_NTZ.ToString(), + DateTime.Parse("9999-12-31 23:59:59.999999"), + DateTime.Parse("9999-12-31 23:59:59.999999") + }; + yield return new object[] + { + "9999-12-31 23:59:59.999999 +1:00", + SFDataType.TIMESTAMP_TZ.ToString(), + null, + DateTime.SpecifyKind(DateTime.Parse("9999-12-31 22:59:59.999999"), DateTimeKind.Utc) + }; + yield return new object[] + { + "9999-12-31 23:59:59.999999 +13:00", + SFDataType.TIMESTAMP_LTZ.ToString(), + null, + DateTime.Parse("9999-12-31 10:59:59.999999").ToLocalTime() + }; + yield return new object[] + { + "0001-01-01 00:00:00", + SFDataType.TIMESTAMP_NTZ.ToString(), + DateTime.Parse("0001-01-01 00:00:00"), + DateTime.Parse("0001-01-01 00:00:00") + }; + yield return new object[] + { + "0001-01-01 00:00:00 -1:00", + SFDataType.TIMESTAMP_TZ.ToString(), + null, + DateTime.SpecifyKind(DateTime.Parse("0001-01-01 01:00:00"), DateTimeKind.Utc) + }; } [Test] @@ -445,6 +480,41 @@ internal static IEnumerable DateTimeOffsetConversionCases() null, DateTimeOffset.Parse("2024-07-11 14:20:05.1234568 -7:00") }; + yield return new object[] + { + "9999-12-31 23:59:59.999999", + SFDataType.TIMESTAMP_NTZ.ToString(), + DateTime.Parse("9999-12-31 23:59:59.999999"), + DateTimeOffset.Parse("9999-12-31 23:59:59.999999Z") + }; + yield return new object[] + { + "9999-12-31 23:59:59.999999 +1:00", + SFDataType.TIMESTAMP_TZ.ToString(), + null, + DateTimeOffset.Parse("9999-12-31 23:59:59.999999 +1:00") + }; + yield return new object[] + { + "9999-12-31 23:59:59.999999 +13:00", + SFDataType.TIMESTAMP_LTZ.ToString(), + null, + DateTimeOffset.Parse("9999-12-31 23:59:59.999999 +13:00") + }; + yield return new object[] + { + "0001-01-01 00:00:00", + SFDataType.TIMESTAMP_NTZ.ToString(), + DateTime.Parse("0001-01-01 00:00:00"), + DateTimeOffset.Parse("0001-01-01 00:00:00Z") + }; + yield return new object[] + { + "0001-01-01 00:00:00 -1:00", + SFDataType.TIMESTAMP_TZ.ToString(), + null, + DateTimeOffset.Parse("0001-01-01 00:00:00 -1:00") + }; } private TimeZoneInfo GetTimeZone(SnowflakeDbConnection connection) diff --git a/Snowflake.Data.Tests/UnitTests/SFBindUploaderTest.cs b/Snowflake.Data.Tests/UnitTests/SFBindUploaderTest.cs index ac5172086..46e5b5b90 100644 --- a/Snowflake.Data.Tests/UnitTests/SFBindUploaderTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFBindUploaderTest.cs @@ -20,7 +20,7 @@ public void TestCsvDataConversionForDate(SFDataType dbType, string input, string { // Arrange var dateExpected = DateTime.Parse(expected); - var check = SFDataConverter.csharpValToSfVal(SFDataType.DATE, dateExpected); + var check = SFDataConverter.CSharpValToSfVal(SFDataType.DATE, dateExpected); Assert.AreEqual(check, input); // Act DateTime dateActual = DateTime.Parse(_bindUploader.GetCSVData(dbType.ToString(), input)); @@ -37,51 +37,60 @@ public void TestCsvDataConversionForTime(SFDataType dbType, string input, string { // Arrange DateTime timeExpected = DateTime.Parse(expected); - var check = SFDataConverter.csharpValToSfVal(SFDataType.TIME, timeExpected); + var check = SFDataConverter.CSharpValToSfVal(SFDataType.TIME, timeExpected); Assert.AreEqual(check, input); // Act DateTime timeActual = DateTime.Parse(_bindUploader.GetCSVData(dbType.ToString(), input)); // Assert Assert.AreEqual(timeExpected, timeActual); } - - [TestCase(SFDataType.TIMESTAMP_LTZ, "39600000000000", "1970-01-01T12:00:00.0000000+01:00")] + + [TestCase(SFDataType.TIMESTAMP_LTZ, "0", "1970-01-01T00:00:00.0000000+00:00")] + [TestCase(SFDataType.TIMESTAMP_LTZ, "39600000000000", "1970-01-01T12:00:00.0000000+01:00")] [TestCase(SFDataType.TIMESTAMP_LTZ, "1341136800000000000", "2012-07-01T12:00:00.0000000+02:00")] [TestCase(SFDataType.TIMESTAMP_LTZ, "352245599987654000", "1981-02-28T23:59:59.9876540+02:00")] [TestCase(SFDataType.TIMESTAMP_LTZ, "1678868249207000000", "2023/03/15T13:17:29.207+05:00")] + [TestCase(SFDataType.TIMESTAMP_LTZ, "253402300799999999900", "9999-12-31T23:59:59.9999999+00:00")] + [TestCase(SFDataType.TIMESTAMP_LTZ, "-62135596800000000000", "0001-01-01T00:00:00.0000000+00:00")] public void TestCsvDataConversionForTimestampLtz(SFDataType dbType, string input, string expected) { // Arrange var timestampExpected = DateTimeOffset.Parse(expected); - var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_LTZ, timestampExpected); + var check = SFDataConverter.CSharpValToSfVal(SFDataType.TIMESTAMP_LTZ, timestampExpected); Assert.AreEqual(check, input); // Act var timestampActual = DateTimeOffset.Parse(_bindUploader.GetCSVData(dbType.ToString(), input)); // Assert Assert.AreEqual(timestampExpected.ToLocalTime(), timestampActual); } - + + [TestCase(SFDataType.TIMESTAMP_TZ, "0 1440", "1970-01-01 00:00:00.000000 +00:00")] [TestCase(SFDataType.TIMESTAMP_TZ, "1341136800000000000 1560", "2012-07-01 12:00:00.000000 +02:00")] [TestCase(SFDataType.TIMESTAMP_TZ, "352245599987654000 1560", "1981-02-28 23:59:59.987654 +02:00")] + [TestCase(SFDataType.TIMESTAMP_TZ, "253402300799999999000 1440", "9999-12-31 23:59:59.999999 +00:00")] + [TestCase(SFDataType.TIMESTAMP_TZ, "-62135596800000000000 1440", "0001-01-01 00:00:00.000000 +00:00")] public void TestCsvDataConversionForTimestampTz(SFDataType dbType, string input, string expected) { // Arrange DateTimeOffset timestampExpected = DateTimeOffset.Parse(expected); - var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_TZ, timestampExpected); + var check = SFDataConverter.CSharpValToSfVal(SFDataType.TIMESTAMP_TZ, timestampExpected); Assert.AreEqual(check, input); // Act DateTimeOffset timestampActual = DateTimeOffset.Parse(_bindUploader.GetCSVData(dbType.ToString(), input)); // Assert Assert.AreEqual(timestampExpected, timestampActual); } - + + [TestCase(SFDataType.TIMESTAMP_NTZ, "0", "1970-01-01 00:00:00.000000")] [TestCase(SFDataType.TIMESTAMP_NTZ, "1341144000000000000", "2012-07-01 12:00:00.000000")] [TestCase(SFDataType.TIMESTAMP_NTZ, "352252799987654000", "1981-02-28 23:59:59.987654")] + [TestCase(SFDataType.TIMESTAMP_NTZ, "253402300799999999000", "9999-12-31 23:59:59.999999")] + [TestCase(SFDataType.TIMESTAMP_NTZ, "-62135596800000000000", "0001-01-01 00:00:00.000000")] public void TestCsvDataConversionForTimestampNtz(SFDataType dbType, string input, string expected) { - // Arrange + // Arrange DateTime timestampExpected = DateTime.Parse(expected); - var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_NTZ, timestampExpected); + var check = SFDataConverter.CSharpValToSfVal(SFDataType.TIMESTAMP_NTZ, timestampExpected); Assert.AreEqual(check, input); // Act DateTime timestampActual = DateTime.Parse(_bindUploader.GetCSVData(dbType.ToString(), input)); diff --git a/Snowflake.Data.Tests/UnitTests/SFDataConverterTest.cs b/Snowflake.Data.Tests/UnitTests/SFDataConverterTest.cs index 65160ac97..7def7ce6a 100755 --- a/Snowflake.Data.Tests/UnitTests/SFDataConverterTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFDataConverterTest.cs @@ -4,6 +4,8 @@ using System; using System.Text; +using Snowflake.Data.Client; +using Snowflake.Data.Tests.Util; namespace Snowflake.Data.Tests.UnitTests { @@ -36,8 +38,8 @@ public void TestConvertBindToSFValFinlandLocale() Thread.CurrentThread.CurrentCulture = ci; - System.Tuple t = - SFDataConverter.csharpTypeValToSfTypeVal(System.Data.DbType.Double, 1.2345); + System.Tuple t = + SFDataConverter.CSharpTypeValToSfTypeVal(System.Data.DbType.Double, 1.2345); Assert.AreEqual("REAL", t.Item1); Assert.AreEqual("1.2345", t.Item2); @@ -109,7 +111,7 @@ public void TestConvertTimeSpan(string inputTimeStr) var tickDiff = val.Ticks; var inputStringAsItComesBackFromDatabase = (tickDiff / 10000000.0m).ToString(CultureInfo.InvariantCulture); inputStringAsItComesBackFromDatabase += inputTimeStr.Substring(8, inputTimeStr.Length - 8); - + // Run the conversion var result = SFDataConverter.ConvertToCSharpVal(ConvertToUTF8Buffer(inputStringAsItComesBackFromDatabase), SFDataType.TIME, typeof(TimeSpan)); @@ -148,7 +150,7 @@ public void TestConvertDate(string inputTimeStr, object kind = null) private void internalTestConvertDate(DateTime dtExpected, DateTime testValue) { - var result = SFDataConverter.csharpTypeValToSfTypeVal(System.Data.DbType.Date, testValue); + var result = SFDataConverter.CSharpTypeValToSfTypeVal(System.Data.DbType.Date, testValue); // Convert result to DateTime for easier interpretation var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); DateTime dtResult = unixEpoch.AddMilliseconds(Int64.Parse(result.Item2)); @@ -326,5 +328,25 @@ public void TestInvalidConversionInvalidDecimal(string s) Assert.Throws(() => SFDataConverter.ConvertToCSharpVal(ConvertToUTF8Buffer(s), SFDataType.FIXED, typeof(decimal))); } + [Test] + [TestCase(SFDataType.TIMESTAMP_LTZ, typeof(DateTime))] + [TestCase(SFDataType.TIMESTAMP_TZ, typeof(DateTime))] + [TestCase(SFDataType.TIMESTAMP_NTZ, typeof(DateTimeOffset))] + [TestCase(SFDataType.TIME, typeof(DateTimeOffset))] + [TestCase(SFDataType.DATE, typeof(DateTimeOffset))] + public void TestInvalidTimestampConversion(SFDataType dataType, Type unsupportedType) + { + object unsupportedObject; + if (unsupportedType == typeof(DateTimeOffset)) + unsupportedObject = new DateTimeOffset(); + else if (unsupportedType == typeof(DateTime)) + unsupportedObject = new DateTime(); + else + unsupportedObject = null; + + Assert.NotNull(unsupportedObject); + SnowflakeDbException ex = Assert.Throws(() => SFDataConverter.CSharpValToSfVal(dataType, unsupportedObject)); + SnowflakeDbExceptionAssert.HasErrorCode(ex, SFError.INVALID_DATA_CONVERSION); + } } } diff --git a/Snowflake.Data.Tests/UnitTests/StructuredTypesTest.cs b/Snowflake.Data.Tests/UnitTests/StructuredTypesTest.cs index 0a91fdab5..cff0c6959 100644 --- a/Snowflake.Data.Tests/UnitTests/StructuredTypesTest.cs +++ b/Snowflake.Data.Tests/UnitTests/StructuredTypesTest.cs @@ -47,6 +47,18 @@ internal static IEnumerable TimeConversionCases() yield return new object[] {"2024-07-11 14:20:05.123456 -7:00", SFDataType.TIMESTAMP_LTZ.ToString(), DateTimeOffset.Parse("2024-07-11 14:20:05.123456 -7:00")}; yield return new object[] {"2024-07-11 14:20:05.123456 -7:00", SFDataType.TIMESTAMP_LTZ.ToString(), DateTime.Parse("2024-07-11 21:20:05.123456").ToLocalTime()}; yield return new object[] {"14:20:05.123456", SFDataType.TIME.ToString(), TimeSpan.Parse("14:20:05.123456")}; + yield return new object[] {"9999-12-31 23:59:59.999999", SFDataType.TIMESTAMP_NTZ.ToString(), DateTime.Parse("9999-12-31 23:59:59.999999")}; + yield return new object[] {"9999-12-31 23:59:59.999999", SFDataType.TIMESTAMP_NTZ.ToString(), DateTimeOffset.Parse("9999-12-31 23:59:59.999999Z")}; + yield return new object[] {"9999-12-31 23:59:59.999999 +1:00", SFDataType.TIMESTAMP_TZ.ToString(), DateTimeOffset.Parse("9999-12-31 23:59:59.999999 +1:00")}; + yield return new object[] {"9999-12-31 23:59:59.999999 +1:00", SFDataType.TIMESTAMP_TZ.ToString(), DateTime.SpecifyKind(DateTime.Parse("9999-12-31 22:59:59.999999"), DateTimeKind.Utc)}; + yield return new object[] {"9999-12-31 23:59:59.999999 +1:00", SFDataType.TIMESTAMP_LTZ.ToString(), DateTimeOffset.Parse("9999-12-31 23:59:59.999999 +1:00")}; + yield return new object[] {"9999-12-31 23:59:59.999999 +13:00", SFDataType.TIMESTAMP_LTZ.ToString(), DateTime.Parse("9999-12-31 10:59:59.999999").ToLocalTime()}; + yield return new object[] {"0001-01-01 00:00:00.123456", SFDataType.TIMESTAMP_NTZ.ToString(), DateTime.Parse("0001-01-01 00:00:00.123456")}; + yield return new object[] {"0001-01-01 00:00:00.123456", SFDataType.TIMESTAMP_NTZ.ToString(), DateTimeOffset.Parse("0001-01-01 00:00:00.123456Z")}; + yield return new object[] {"0001-01-01 00:00:00.123456 -1:00", SFDataType.TIMESTAMP_TZ.ToString(), DateTimeOffset.Parse("0001-01-01 00:00:00.123456 -1:00")}; + yield return new object[] {"0001-01-01 00:00:00.123456 -1:00", SFDataType.TIMESTAMP_TZ.ToString(), DateTime.SpecifyKind(DateTime.Parse("0001-01-01 01:00:00.123456"), DateTimeKind.Utc)}; + yield return new object[] {"0001-01-01 00:00:00.123456 -1:00", SFDataType.TIMESTAMP_LTZ.ToString(), DateTimeOffset.Parse("0001-01-01 00:00:00.123456 -1:00")}; + yield return new object[] {"0001-01-01 00:00:00.123456 -13:00", SFDataType.TIMESTAMP_LTZ.ToString(), DateTime.Parse("0001-01-01 13:00:00.123456").ToLocalTime()}; } } } diff --git a/Snowflake.Data/Client/SnowflakeDbCommand.cs b/Snowflake.Data/Client/SnowflakeDbCommand.cs index b52d53643..68d3dccb0 100755 --- a/Snowflake.Data/Client/SnowflakeDbCommand.cs +++ b/Snowflake.Data/Client/SnowflakeDbCommand.cs @@ -393,7 +393,7 @@ private static Dictionary convertToBindList(List typeAndVal = SFDataConverter - .csharpTypeValToSfTypeVal(parameter.DbType, val); + .CSharpTypeValToSfTypeVal(parameter.DbType, val); bindingType = typeAndVal.Item1; vals.Add(typeAndVal.Item2); @@ -401,7 +401,7 @@ private static Dictionary convertToBindList(List convertToBindList(List typeAndVal = SFDataConverter - .csharpTypeValToSfTypeVal(parameter.DbType, parameter.Value); + .CSharpTypeValToSfTypeVal(parameter.DbType, parameter.Value); bindingType = typeAndVal.Item1; bindingVal = typeAndVal.Item2; } else { bindingType = parameter.SFDataType.ToString(); - bindingVal = SFDataConverter.csharpValToSfVal(parameter.SFDataType, parameter.Value); + bindingVal = SFDataConverter.CSharpValToSfVal(parameter.SFDataType, parameter.Value); } } diff --git a/Snowflake.Data/Core/ArrowResultSet.cs b/Snowflake.Data/Core/ArrowResultSet.cs index a3a6e2628..178531eaf 100755 --- a/Snowflake.Data/Core/ArrowResultSet.cs +++ b/Snowflake.Data/Core/ArrowResultSet.cs @@ -392,7 +392,7 @@ internal override string GetString(int ordinal) return ret; case DateTime ret: if (type == SFDataType.DATE) - return SFDataConverter.toDateString(ret, sfResultSetMetaData.dateOutputFormat); + return SFDataConverter.ToDateString(ret, sfResultSetMetaData.dateOutputFormat); break; } diff --git a/Snowflake.Data/Core/SFBindUploader.cs b/Snowflake.Data/Core/SFBindUploader.cs index 6268c724c..400c3b0c9 100644 --- a/Snowflake.Data/Core/SFBindUploader.cs +++ b/Snowflake.Data/Core/SFBindUploader.cs @@ -251,26 +251,38 @@ internal string GetCSVData(string sType, string sValue) return '"' + sValue.Replace("\"", "\"\"") + '"'; return sValue; case "DATE": - long msFromEpoch = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ms] from Epoch + long msFromEpoch = long.Parse(sValue); // SFDateConverter.CSharpValToSfVal provides in [ms] from Epoch DateTime date = epoch.AddMilliseconds(msFromEpoch); return date.ToShortDateString(); case "TIME": - long nsSinceMidnight = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ns] from Midnight + long nsSinceMidnight = long.Parse(sValue); // SFDateConverter.CSharpValToSfVal provides in [ns] from Midnight DateTime time = epoch.AddTicks(nsSinceMidnight/100); return time.ToString("HH:mm:ss.fffffff"); case "TIMESTAMP_LTZ": - long nsFromEpochLtz = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ns] from Epoch - DateTime ltz = epoch.AddTicks(nsFromEpochLtz/100); + long ticksFromEpochLtz = + long.TryParse(sValue, out var nsLtz) + ? nsLtz / 100 + : (long)(decimal.Parse(sValue) / 100); + + DateTime ltz = epoch.AddTicks(ticksFromEpochLtz); return ltz.ToLocalTime().ToString("O"); // ISO 8601 format case "TIMESTAMP_NTZ": - long nsFromEpochNtz = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ns] from Epoch - DateTime ntz = epoch.AddTicks(nsFromEpochNtz/100); + long ticksFromEpochNtz = + long.TryParse(sValue, out var nsNtz) + ? nsNtz / 100 + : (long)(decimal.Parse(sValue) / 100); + + DateTime ntz = epoch.AddTicks(ticksFromEpochNtz); return ntz.ToString("yyyy-MM-dd HH:mm:ss.fffffff"); case "TIMESTAMP_TZ": string[] tstzString = sValue.Split(' '); - long nsFromEpochTz = long.Parse(tstzString[0]); // SFDateConverter provides in [ns] from Epoch + long ticksFromEpochTz = + long.TryParse(tstzString[0], out var nsTz) + ? nsTz / 100 + : (long)(decimal.Parse(tstzString[0]) / 100); + int timeZoneOffset = int.Parse(tstzString[1]) - 1440; // SFDateConverter provides in minutes increased by 1440m - DateTime timestamp = epoch.AddTicks(nsFromEpochTz/100).AddMinutes(timeZoneOffset); + DateTime timestamp = epoch.AddTicks(ticksFromEpochTz).AddMinutes(timeZoneOffset); TimeSpan offset = TimeSpan.FromMinutes(timeZoneOffset); DateTimeOffset tzDateTimeOffset = new DateTimeOffset(timestamp.Ticks, offset); return tzDateTimeOffset.ToString("yyyy-MM-dd HH:mm:ss.fffffff zzz"); diff --git a/Snowflake.Data/Core/SFDataConverter.cs b/Snowflake.Data/Core/SFDataConverter.cs index 90e956314..619976400 100755 --- a/Snowflake.Data/Core/SFDataConverter.cs +++ b/Snowflake.Data/Core/SFDataConverter.cs @@ -152,7 +152,7 @@ private static DateTime ConvertToDateTime(UTF8Buffer srcVal, SFDataType srcType) { case SFDataType.DATE: long srcValLong = FastParser.FastParseInt64(srcVal.Buffer, srcVal.offset, srcVal.length); - return DateTime.SpecifyKind(UnixEpoch.AddDays(srcValLong), DateTimeKind.Unspecified);; + return DateTime.SpecifyKind(UnixEpoch.AddDays(srcValLong), DateTimeKind.Unspecified); case SFDataType.TIME: case SFDataType.TIMESTAMP_NTZ: @@ -240,7 +240,7 @@ private static long GetTicksFromSecondAndNanosecond(UTF8Buffer srcVal) } - internal static Tuple csharpTypeValToSfTypeVal(DbType srcType, object srcVal) + internal static Tuple CSharpTypeValToSfTypeVal(DbType srcType, object srcVal) { SFDataType destType; string destVal; @@ -300,7 +300,7 @@ internal static Tuple csharpTypeValToSfTypeVal(DbType srcType, o default: throw new SnowflakeDbException(SFError.UNSUPPORTED_DOTNET_TYPE, srcType); } - destVal = csharpValToSfVal(destType, srcVal); + destVal = CSharpValToSfVal(destType, srcVal); return Tuple.Create(destType.ToString(), destVal); } @@ -323,7 +323,7 @@ internal static byte[] HexToBytes(string hex) return bytes; } - internal static string csharpValToSfVal(SFDataType sfDataType, object srcVal) + internal static string CSharpValToSfVal(SFDataType sfDataType, object srcVal) { string destVal = null; @@ -331,18 +331,6 @@ internal static string csharpValToSfVal(SFDataType sfDataType, object srcVal) { switch (sfDataType) { - case SFDataType.TIMESTAMP_LTZ: - if (srcVal.GetType() != typeof(DateTimeOffset)) - { - throw new SnowflakeDbException(SFError.INVALID_DATA_CONVERSION, srcVal, - srcVal.GetType().ToString(), SFDataType.TIMESTAMP_LTZ.ToString()); - } - else - { - destVal = ((long)(((DateTimeOffset)srcVal).UtcTicks - UnixEpoch.Ticks) * 100).ToString(); - } - break; - case SFDataType.FIXED: case SFDataType.BOOLEAN: case SFDataType.REAL: @@ -359,9 +347,8 @@ internal static string csharpValToSfVal(SFDataType sfDataType, object srcVal) else { DateTime srcDt = ((DateTime)srcVal); - long nanoSinceMidNight = (long)(srcDt.Ticks - srcDt.Date.Ticks) * 100L; - - destVal = nanoSinceMidNight.ToString(); + var tickDiff = srcDt.Ticks - srcDt.Date.Ticks; + destVal = TicksToNanoSecondsString(tickDiff); } break; @@ -380,6 +367,19 @@ internal static string csharpValToSfVal(SFDataType sfDataType, object srcVal) } break; + case SFDataType.TIMESTAMP_LTZ: + if (srcVal.GetType() != typeof(DateTimeOffset)) + { + throw new SnowflakeDbException(SFError.INVALID_DATA_CONVERSION, srcVal, + srcVal.GetType().ToString(), SFDataType.TIMESTAMP_LTZ.ToString()); + } + else + { + var tickDiff = ((DateTimeOffset)srcVal).UtcTicks - UnixEpoch.Ticks; + destVal = TicksToNanoSecondsString(tickDiff); + } + break; + case SFDataType.TIMESTAMP_NTZ: if (srcVal.GetType() != typeof(DateTime)) { @@ -391,7 +391,7 @@ internal static string csharpValToSfVal(SFDataType sfDataType, object srcVal) DateTime srcDt = (DateTime)srcVal; var diff = srcDt.Subtract(UnixEpoch); var tickDiff = diff.Ticks; - destVal = $"{tickDiff}00"; // Cannot multiple tickDiff by 100 because long might overflow. + destVal = TicksToNanoSecondsString(tickDiff); } break; @@ -404,8 +404,8 @@ internal static string csharpValToSfVal(SFDataType sfDataType, object srcVal) else { DateTimeOffset dtOffset = (DateTimeOffset)srcVal; - destVal = String.Format("{0} {1}", (dtOffset.UtcTicks - UnixEpoch.Ticks) * 100L, - dtOffset.Offset.TotalMinutes + 1440); + var tickDiff = dtOffset.UtcTicks - UnixEpoch.Ticks; + destVal = $"{TicksToNanoSecondsString(tickDiff)} {dtOffset.Offset.TotalMinutes + 1440}"; } break; @@ -429,7 +429,9 @@ internal static string csharpValToSfVal(SFDataType sfDataType, object srcVal) return destVal; } - internal static string toDateString(DateTime date, string formatter) + private static string TicksToNanoSecondsString(long tickDiff) => tickDiff == 0 ? "0" : $"{tickDiff}00"; + + internal static string ToDateString(DateTime date, string formatter) { // change formatter from "YYYY-MM-DD" to "yyyy-MM-dd" formatter = formatter.Replace("Y", "y").Replace("m", "M").Replace("D", "d"); diff --git a/Snowflake.Data/Core/SFResultSet.cs b/Snowflake.Data/Core/SFResultSet.cs index a7586f2c3..e81db8c14 100755 --- a/Snowflake.Data/Core/SFResultSet.cs +++ b/Snowflake.Data/Core/SFResultSet.cs @@ -283,7 +283,7 @@ internal override string GetString(int ordinal) var val = GetValue(ordinal); if (val == DBNull.Value) return null; - return SFDataConverter.toDateString((DateTime)val, sfResultSetMetaData.dateOutputFormat); + return SFDataConverter.ToDateString((DateTime)val, sfResultSetMetaData.dateOutputFormat); default: return GetObjectInternal(ordinal).SafeToString();