SNOW-1729244 support for large timestamps

Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs

@@ -884,13 +884,20 @@ public void TestExplicitDbTypeAssignmentForArrayValue()
         [TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
         */
         // Session TimeZone cases
-        [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
+        [TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
         [TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")]
+        [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
+        [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")]
         public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType, SFDataType columnType, Int32? columnPrecision, DbType bindingType, string comparisonFormat, string timeZone)
         {
             // Arrange
-            var timestamp = "2023/03/15 13:17:29.207 +05:00"; // 08:17:29.207 UTC
-            var expected = ExpectedTimestampWrapper.From(timestamp, columnType);
+            string[] timestamps =
+            {
+                "2023/03/15 13:17:29.207 +05:00",
+                "9999/12/30 23:24:25.987 +07:00",
+                "0001/01/02 02:06:07.000 -04:00"
+            };
+            var expected = ExpectedTimestampWrapper.From(timestamps, columnType);
             var columnWithPrecision = ColumnTypeWithPrecision(columnType, columnPrecision);
             var testCase = $"ResultFormat={resultFormat}, TableType={tableType}, ColumnType={columnWithPrecision}, BindingType={bindingType}, ComparisonFormat={comparisonFormat}";
             var bindingThreshold = 65280; // when exceeded enforces bindings via file on stage
@@ -906,24 +913,34 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
                 if (!timeZone.IsNullOrEmpty()) // Driver ignores this setting and relies on local environment timezone
                     conn.ExecuteNonQuery($"alter session set TIMEZONE = '{timeZone}'");
 
+                // prepare initial column
+                var columns = new List<String> { "id number(10,0) not null primary key" };
+                var sql_columns = "id";
+                var sql_values = "?";
+
+                // prepare additional columns
+                for (int i = 1; i <= timestamps.Length; ++i)
+                {
+                    columns.Add($"ts_{i} {columnWithPrecision}");
+                    sql_columns += $",ts_{i}";
+                    sql_values += ",?";
+                }
+
                 CreateOrReplaceTable(conn,
                     TableName,
                     tableType.TableDDLCreationPrefix(),
-                    new[] {
-                        "id number(10,0) not null primary key", // necessary only for HYBRID tables
-                        $"ts {columnWithPrecision}"
-                    },
+                    columns,
                     tableType.TableDDLCreationFlags());
 
                 // Act+Assert
-                var sqlInsert = $"insert into {TableName} (id, ts) values (?, ?)";
+                var sqlInsert = $"insert into {TableName} ({sql_columns}) values ({sql_values})";
                 InsertSingleRecord(conn, sqlInsert, bindingType, 1, expected);
                 InsertMultipleRecords(conn, sqlInsert, bindingType, 2, expected, smallBatchRowCount, false);
                 InsertMultipleRecords(conn, sqlInsert, bindingType, smallBatchRowCount+2, expected, bigBatchRowCount, true);
 
                 // Assert
                 var row = 0;
-                using (var select = conn.CreateCommand($"select id, ts from {TableName} order by id"))
+                using (var select = conn.CreateCommand($"select {sql_columns} from {TableName} order by id"))
                 {
                     s_logger.Debug(select.CommandText);
                     var reader = select.ExecuteReader();
@@ -932,7 +949,11 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
                         ++row;
                         string faultMessage = $"Mismatch for row: {row}, {testCase}";
                         Assert.AreEqual(row, reader.GetInt32(0));
-                        expected.AssertEqual(reader.GetValue(1), comparisonFormat, faultMessage);
+
+                        for (int i = 0; i < timestamps.Length; ++i)
+                        {
+                            expected.AssertEqual(reader.GetValue(i + 1), comparisonFormat, faultMessage, i);
+                        }
                     }
                 }
                 Assert.AreEqual(1+smallBatchRowCount+bigBatchRowCount, row);
@@ -947,12 +968,24 @@ private void InsertSingleRecord(IDbConnection conn, string sqlInsert, DbType bin
                 insert.Add("1", DbType.Int32, identifier);
                 if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType()))
                 {
-                    var parameter = (SnowflakeDbParameter)insert.Add("2", binding, ts.GetDateTimeOffset());
-                    parameter.SFDataType = ts.ExpectedColumnType();
+                    var dateTimeOffsets = ts.GetDateTimeOffsets();
+                    for (int i = 0; i < dateTimeOffsets.Length; ++i)
+                    {
+                        var parameterName = (i + 2).ToString();
+                        var parameterValue = dateTimeOffsets[i];
+                        var parameter = insert.Add(parameterName, binding, parameterValue);
+                        parameter.SFDataType = ts.ExpectedColumnType();
+                    }
                 }
                 else
                 {
-                    insert.Add("2", binding, ts.GetDateTime());
+                    var dateTimes = ts.GetDateTimes();
+                    for (int i = 0; i < dateTimes.Length; ++i)
+                    {
+                        var parameterName = (i + 2).ToString();
+                        var parameterValue = dateTimes[i];
+                        insert.Add(parameterName, binding, parameterValue);
+                    }
                 }
 
                 // Act
@@ -973,12 +1006,25 @@ private void InsertMultipleRecords(IDbConnection conn, string sqlInsert, DbType
                 insert.Add("1", DbType.Int32, Enumerable.Range(initialIdentifier, rowsCount).ToArray());
                 if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType()))
                 {
-                    var parameter = (SnowflakeDbParameter)insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTimeOffset(), rowsCount).ToArray());
-                    parameter.SFDataType = ts.ExpectedColumnType();
+                    var dateTimeOffsets = ts.GetDateTimeOffsets();
+                    for (int i = 0; i < dateTimeOffsets.Length; ++i)
+                    {
+                        var parameterName = (i + 2).ToString();
+                        var parameterValue = Enumerable.Repeat(dateTimeOffsets[i], rowsCount).ToArray();
+                        var parameter = insert.Add(parameterName, binding, parameterValue);
+                        parameter.SFDataType = ts.ExpectedColumnType();
+                    }
+
                 }
                 else
                 {
-                    insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTime(), rowsCount).ToArray());
+                    var dateTimes = ts.GetDateTimes();
+                    for (int i = 0; i < dateTimes.Length; ++i)
+                    {
+                        var parameterName = (i + 2).ToString();
+                        var parameterValue = Enumerable.Repeat(dateTimes[i], rowsCount).ToArray();
+                        insert.Add(parameterName, binding, parameterValue);
+                    }
                 }
 
                 // Act
@@ -1001,56 +1047,63 @@ private static string ColumnTypeWithPrecision(SFDataType columnType, Int32? colu
     class ExpectedTimestampWrapper
     {
         private readonly SFDataType _columnType;
-        private readonly DateTime? _expectedDateTime;
-        private readonly DateTimeOffset? _expectedDateTimeOffset;
+        private readonly DateTime[]? _expectedDateTimes;
+        private readonly DateTimeOffset[]? _expectedDateTimeOffsets;
 
-        internal static ExpectedTimestampWrapper From(string timestampWithTimeZone, SFDataType columnType)
+        internal static ExpectedTimestampWrapper From(string[] timestampsWithTimeZone, SFDataType columnType)
         {
             if (IsOffsetType(columnType))
             {
-                var dateTimeOffset = DateTimeOffset.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture);
-                return new ExpectedTimestampWrapper(dateTimeOffset, columnType);
+                var dateTimeOffsets =
+                    timestampsWithTimeZone
+                        .Select(ts => DateTimeOffset.ParseExact(ts, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture))
+                        .ToArray();
+                return new ExpectedTimestampWrapper(dateTimeOffsets, columnType);
             }
 
-            var dateTime = DateTime.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture);
-            return new ExpectedTimestampWrapper(dateTime, columnType);
+            var dateTimes =
+                timestampsWithTimeZone
+                    .Select(ts => DateTime.ParseExact(ts, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture))
+                    .ToArray();
+
+            return new ExpectedTimestampWrapper(dateTimes, columnType);
         }
 
-        private ExpectedTimestampWrapper(DateTime dateTime, SFDataType columnType)
+        private ExpectedTimestampWrapper(DateTime[] dateTimes, SFDataType columnType)
         {
-            _expectedDateTime = dateTime;
-            _expectedDateTimeOffset = null;
+            _expectedDateTimes = dateTimes;
+            _expectedDateTimeOffsets = null;
             _columnType = columnType;
         }
 
-        private ExpectedTimestampWrapper(DateTimeOffset dateTimeOffset, SFDataType columnType)
+        private ExpectedTimestampWrapper(DateTimeOffset[] dateTimeOffsets, SFDataType columnType)
         {
-            _expectedDateTimeOffset = dateTimeOffset;
-            _expectedDateTime = null;
+            _expectedDateTimeOffsets = dateTimeOffsets;
+            _expectedDateTimes = null;
             _columnType = columnType;
         }
 
         internal SFDataType ExpectedColumnType() => _columnType;
 
-        internal void AssertEqual(object actual, string comparisonFormat, string faultMessage)
+        internal void AssertEqual(object actual, string comparisonFormat, string faultMessage, int index)
         {
             switch (_columnType)
             {
                 case SFDataType.TIMESTAMP_TZ:
-                    Assert.AreEqual(GetDateTimeOffset().ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage);
+                    Assert.AreEqual(GetDateTimeOffsets()[index].ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage);
                     break;
                 case SFDataType.TIMESTAMP_LTZ:
-                    Assert.AreEqual(GetDateTimeOffset().ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage);
+                    Assert.AreEqual(GetDateTimeOffsets()[index].ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage);
                     break;
                 default:
-                    Assert.AreEqual(GetDateTime().ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage);
+                    Assert.AreEqual(GetDateTimes()[index].ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage);
                     break;
             }
         }
 
-        internal DateTime GetDateTime() => _expectedDateTime ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");
+        internal DateTime[] GetDateTimes() => _expectedDateTimes ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");
 
-        internal DateTimeOffset GetDateTimeOffset() => _expectedDateTimeOffset ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");
+        internal DateTimeOffset[] GetDateTimeOffsets() => _expectedDateTimeOffsets ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");
 
         internal static bool IsOffsetType(SFDataType type) => type == SFDataType.TIMESTAMP_LTZ || type == SFDataType.TIMESTAMP_TZ;
     }

Snowflake.Data.Tests/UnitTests/SFBindUploaderTest.cs

@@ -44,11 +44,14 @@ public void TestCsvDataConversionForTime(SFDataType dbType, string input, string
             // Assert
             Assert.AreEqual(timeExpected, timeActual);
         }
-
-        [TestCase(SFDataType.TIMESTAMP_LTZ, "39600000000000", "1970-01-01T12:00:00.0000000+01:00")] 
+
+        [TestCase(SFDataType.TIMESTAMP_LTZ, "0", "1970-01-01T00:00:00.0000000+00:00")]
+        [TestCase(SFDataType.TIMESTAMP_LTZ, "39600000000000", "1970-01-01T12:00:00.0000000+01:00")]
         [TestCase(SFDataType.TIMESTAMP_LTZ, "1341136800000000000", "2012-07-01T12:00:00.0000000+02:00")]
         [TestCase(SFDataType.TIMESTAMP_LTZ, "352245599987654000", "1981-02-28T23:59:59.9876540+02:00")]
         [TestCase(SFDataType.TIMESTAMP_LTZ, "1678868249207000000", "2023/03/15T13:17:29.207+05:00")]
+        [TestCase(SFDataType.TIMESTAMP_LTZ, "253402300799999999900", "9999-12-31T23:59:59.9999999+00:00")]
+        [TestCase(SFDataType.TIMESTAMP_LTZ, "-62135596800000000000", "0001-01-01T00:00:00.0000000+00:00")]
         public void TestCsvDataConversionForTimestampLtz(SFDataType dbType, string input, string expected)
         {
             // Arrange
@@ -60,9 +63,12 @@ public void TestCsvDataConversionForTimestampLtz(SFDataType dbType, string input
             // Assert
             Assert.AreEqual(timestampExpected.ToLocalTime(), timestampActual);
         }
-
+
+        [TestCase(SFDataType.TIMESTAMP_TZ, "0 1440", "1970-01-01 00:00:00.000000 +00:00")]
         [TestCase(SFDataType.TIMESTAMP_TZ, "1341136800000000000 1560", "2012-07-01 12:00:00.000000 +02:00")]
         [TestCase(SFDataType.TIMESTAMP_TZ, "352245599987654000 1560", "1981-02-28 23:59:59.987654 +02:00")]
+        [TestCase(SFDataType.TIMESTAMP_TZ, "253402300799999999000 1440", "9999-12-31 23:59:59.999999 +00:00")]
+        [TestCase(SFDataType.TIMESTAMP_TZ, "-62135596800000000000 1440", "0001-01-01 00:00:00.000000 +00:00")]
         public void TestCsvDataConversionForTimestampTz(SFDataType dbType, string input, string expected)
         {
             // Arrange
@@ -74,12 +80,15 @@ public void TestCsvDataConversionForTimestampTz(SFDataType dbType, string input,
             // Assert
             Assert.AreEqual(timestampExpected, timestampActual);
         }
-
+
+        [TestCase(SFDataType.TIMESTAMP_NTZ, "0", "1970-01-01 00:00:00.000000")]
         [TestCase(SFDataType.TIMESTAMP_NTZ, "1341144000000000000", "2012-07-01 12:00:00.000000")]
         [TestCase(SFDataType.TIMESTAMP_NTZ, "352252799987654000", "1981-02-28 23:59:59.987654")]
+        [TestCase(SFDataType.TIMESTAMP_NTZ, "253402300799999999000", "9999-12-31 23:59:59.999999")]
+        [TestCase(SFDataType.TIMESTAMP_NTZ, "-62135596800000000000", "0001-01-01 00:00:00.000000")]
         public void TestCsvDataConversionForTimestampNtz(SFDataType dbType, string input, string expected)
         {
-            // Arrange 
+            // Arrange
             DateTime timestampExpected = DateTime.Parse(expected);
             var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_NTZ, timestampExpected);
             Assert.AreEqual(check, input);

Snowflake.Data/Core/SFBindUploader.cs

@@ -259,18 +259,30 @@ internal string GetCSVData(string sType, string sValue)
                     DateTime time = epoch.AddTicks(nsSinceMidnight/100);
                     return time.ToString("HH:mm:ss.fffffff");
                 case "TIMESTAMP_LTZ":
-                    long nsFromEpochLtz = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ns] from Epoch
-                    DateTime ltz = epoch.AddTicks(nsFromEpochLtz/100);
+                    long ticksFromEpochLtz =
+                        long.TryParse(sValue, out var nssLtz) ?
+                            nssLtz / 100 :
+                            (long)(decimal.Parse(sValue) / 100);
+
+                    DateTime ltz = epoch.AddTicks(ticksFromEpochLtz);
                     return ltz.ToLocalTime().ToString("O"); // ISO 8601 format
                 case "TIMESTAMP_NTZ":
-                    long nsFromEpochNtz = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ns] from Epoch
-                    DateTime ntz = epoch.AddTicks(nsFromEpochNtz/100);
+                    long ticksFromEpochNtz =
+                        long.TryParse(sValue, out var nsNtz) ?
+                            nsNtz / 100 :
+                            (long)(decimal.Parse(sValue) / 100);
+
+                    DateTime ntz = epoch.AddTicks(ticksFromEpochNtz);
                     return ntz.ToString("yyyy-MM-dd HH:mm:ss.fffffff");
                 case "TIMESTAMP_TZ":
                     string[] tstzString = sValue.Split(' ');
-                    long nsFromEpochTz = long.Parse(tstzString[0]); // SFDateConverter provides in [ns] from Epoch
+                    long ticksFromEpochTz =
+                        long.TryParse(tstzString[0], out var nsTz) ?
+                            nsTz / 100 :
+                            (long)(decimal.Parse(tstzString[0]) / 100);
+
                     int timeZoneOffset = int.Parse(tstzString[1]) - 1440; // SFDateConverter provides in minutes increased by 1440m
-                    DateTime timestamp = epoch.AddTicks(nsFromEpochTz/100).AddMinutes(timeZoneOffset);
+                    DateTime timestamp = epoch.AddTicks(ticksFromEpochTz).AddMinutes(timeZoneOffset);
                     TimeSpan offset = TimeSpan.FromMinutes(timeZoneOffset);
                     DateTimeOffset tzDateTimeOffset = new DateTimeOffset(timestamp.Ticks, offset);
                     return tzDateTimeOffset.ToString("yyyy-MM-dd HH:mm:ss.fffffff zzz");