diff --git a/Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs index e222e5892..00a1857a2 100755 --- a/Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs @@ -1,25 +1,31 @@ /* - * Copyright (c) 2012-2019 Snowflake Computing Inc. All rights reserved. + * Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. */ using System; using System.Data; +using System.Linq; +using Microsoft.IdentityModel.Tokens; using Newtonsoft.Json; +using Snowflake.Data.Log; +using NUnit.Framework; +using Snowflake.Data.Client; +using Snowflake.Data.Core; +using System.Text; +using System.Globalization; +using System.Collections.Generic; +using Snowflake.Data.Tests.Util; namespace Snowflake.Data.Tests.IntegrationTests { - using NUnit.Framework; - using Snowflake.Data.Client; - using Snowflake.Data.Core; - using System.Text; - using System.Globalization; - using System.Collections.Generic; [TestFixture] class SFBindTestIT : SFBaseTest { + private static readonly SFLogger s_logger = SFLoggerFactory.GetLogger(); + [Test] - public void testArrayBind() + public void TestArrayBind() { using (IDbConnection conn = new SnowflakeDbConnection()) @@ -59,7 +65,7 @@ public void testArrayBind() } [Test] - public void testBindNullValue() + public void TestBindNullValue() { using (SnowflakeDbConnection dbConnection = new SnowflakeDbConnection()) { @@ -196,7 +202,7 @@ public void testBindNullValue() } [Test] - public void testBindValue() + public void TestBindValue() { using (SnowflakeDbConnection dbConnection = new SnowflakeDbConnection()) { @@ -313,7 +319,7 @@ public void testBindValue() command.CommandText = $"insert into {TableName}(stringData) values(:p0)"; param.Value = DBNull.Value; command.Parameters.Add(param); - int rowsInserted = command.ExecuteNonQuery(); + command.ExecuteNonQuery(); } catch (SnowflakeDbException e) { @@ -347,7 +353,7 @@ public void testBindValue() } [Test] - public void testBindValueWithSFDataType() + public void TestBindValueWithSFDataType() { using (SnowflakeDbConnection dbConnection = new SnowflakeDbConnection()) { @@ -440,7 +446,7 @@ public void testBindValueWithSFDataType() command.CommandText = $"insert into {TableName}(unsupportedType) values(:p0)"; param.Value = DBNull.Value; command.Parameters.Add(param); - int rowsInserted = command.ExecuteNonQuery(); + command.ExecuteNonQuery(); } catch (SnowflakeDbException e) { @@ -468,7 +474,7 @@ public void testBindValueWithSFDataType() } [Test] - public void testParameterCollection() + public void TestParameterCollection() { using (IDbConnection conn = new SnowflakeDbConnection()) { @@ -524,7 +530,7 @@ public void testParameterCollection() } [Test] - public void testPutArrayBind() + public void TestPutArrayBind() { using (IDbConnection conn = new SnowflakeDbConnection()) { @@ -646,10 +652,6 @@ public void testPutArrayBind() cmd.CommandText = $"SELECT * FROM {TableName}"; IDataReader reader = cmd.ExecuteReader(); Assert.IsTrue(reader.Read()); - - //cmd.CommandText = "drop table if exists testPutArrayBind"; - //cmd.ExecuteNonQuery(); - } conn.Close(); @@ -657,7 +659,7 @@ public void testPutArrayBind() } [Test] - public void testPutArrayBindWorkDespiteOtTypeNameHandlingAuto() + public void TestPutArrayBindWorkDespiteOtTypeNameHandlingAuto() { JsonConvert.DefaultSettings = () => new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.Auto @@ -729,7 +731,7 @@ public void testPutArrayBindWorkDespiteOtTypeNameHandlingAuto() } [Test] - public void testPutArrayBind1() + public void TestPutArrayIntegerBind() { using (IDbConnection conn = new SnowflakeDbConnection()) { @@ -771,7 +773,7 @@ public void testPutArrayBind1() } [Test] - public void testExplicitDbTypeAssignmentForSimpleValue() + public void TestExplicitDbTypeAssignmentForSimpleValue() { using (IDbConnection conn = new SnowflakeDbConnection()) @@ -803,7 +805,7 @@ public void testExplicitDbTypeAssignmentForSimpleValue() } [Test] - public void testExplicitDbTypeAssignmentForArrayValue() + public void TestExplicitDbTypeAssignmentForArrayValue() { using (IDbConnection conn = new SnowflakeDbConnection()) @@ -833,5 +835,223 @@ public void testExplicitDbTypeAssignmentForArrayValue() conn.Close(); } } + + private const string FormatYmd = "yyyy/MM/dd"; + private const string FormatHms = "HH\\:mm\\:ss"; + private const string FormatHmsf = "HH\\:mm\\:ss\\.fff"; + private const string FormatYmdHms = "yyyy/MM/dd HH\\:mm\\:ss"; + private const string FormatYmdHmsZ = "yyyy/MM/dd HH\\:mm\\:ss zzz"; + + // STANDARD Tables + [TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.DATE, null, DbType.Date, FormatYmd, null)] + [TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIME, null, DbType.Time, FormatHms, null)] + [TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIME, 6, DbType.Time, FormatHmsf, null)] + [TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)] + [TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] + [TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.DATE, null, DbType.Date, FormatYmd, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIME, null, DbType.Time, FormatHms, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIME, 6, DbType.Time, FormatHmsf, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] + /* TODO: Enable when features available on the automated tests environment + // HYBRID Tables + [TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.DATE, null, DbType.Date, FormatYmd, null)] + [TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.TIME, null, DbType.Time, FormatHms, null)] + [TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.TIME, 6, DbType.Time, FormatHmsf, null)] + [TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)] + [TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] + [TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Hybrid, SFDataType.DATE, null, DbType.Date, FormatYmd, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Hybrid, SFDataType.TIME, null, DbType.Time, FormatHms, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Hybrid, SFDataType.TIME, 6, DbType.Time, FormatHmsf, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Hybrid, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Hybrid, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Hybrid, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] + // ICEBERG Tables; require env variables: ICEBERG_EXTERNAL_VOLUME, ICEBERG_CATALOG, ICEBERG_BASE_LOCATION. + [TestCase(ResultFormat.JSON, SFTableType.Iceberg, SFDataType.DATE, null, DbType.Date, FormatYmd, null)] + [TestCase(ResultFormat.JSON, SFTableType.Iceberg, SFDataType.TIME, null, DbType.Time, FormatHms, null)] + [TestCase(ResultFormat.JSON, SFTableType.Iceberg, SFDataType.TIME, 6, DbType.Time, FormatHmsf, null)] + [TestCase(ResultFormat.JSON, SFTableType.Iceberg, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)] + // [TestCase(ResultFormat.JSON, SFTableType.Iceberg, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] // Unsupported data type 'TIMESTAMP_TZ(6)' for iceberg tables + [TestCase(ResultFormat.JSON, SFTableType.Iceberg, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.DATE, null, DbType.Date, FormatYmd, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIME, null, DbType.Time, FormatHms, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIME, 6, DbType.Time, FormatHmsf, null)] + [TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)] + // [TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTime, FormatYmdHmsZ, null)] // Unsupported data type 'TIMESTAMP_TZ(6)' for iceberg tables + [TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] + */ + // Session TimeZone cases + [TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")] + [TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")] + public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType, SFDataType columnType, Int32? columnPrecision, DbType bindingType, string comparisonFormat, string timeZone) + { + // Arrange + var timestamp = "2023/03/15 13:17:29.207 +05:00"; // 08:17:29.207 UTC + var expected = ExpectedTimestampWrapper.From(timestamp, columnType); + var columnWithPrecision = ColumnTypeWithPrecision(columnType, columnPrecision); + var testCase = $"ResultFormat={resultFormat}, TableType={tableType}, ColumnType={columnWithPrecision}, BindingType={bindingType}, ComparisonFormat={comparisonFormat}"; + var bindingThreshold = 65280; // when exceeded enforces bindings via file on stage + var smallBatchRowCount = 2; + var bigBatchRowCount = bindingThreshold / 2; + s_logger.Info(testCase); + + using (IDbConnection conn = new SnowflakeDbConnection(ConnectionString)) + { + conn.Open(); + + conn.ExecuteNonQuery($"alter session set DOTNET_QUERY_RESULT_FORMAT = {resultFormat}"); + if (!timeZone.IsNullOrEmpty()) // Driver ignores this setting and relies on local environment timezone + conn.ExecuteNonQuery($"alter session set TIMEZONE = '{timeZone}'"); + + CreateOrReplaceTable(conn, + TableName, + tableType.TableDDLCreationPrefix(), + new[] { + "id number(10,0) not null primary key", // necessary only for HYBRID tables + $"ts {columnWithPrecision}" + }, + tableType.TableDDLCreationFlags()); + + // Act+Assert + var sqlInsert = $"insert into {TableName} (id, ts) values (?, ?)"; + InsertSingleRecord(conn, sqlInsert, bindingType, 1, expected); + InsertMultipleRecords(conn, sqlInsert, bindingType, 2, expected, smallBatchRowCount, false); + InsertMultipleRecords(conn, sqlInsert, bindingType, smallBatchRowCount+2, expected, bigBatchRowCount, true); + + // Assert + var row = 0; + using (var select = conn.CreateCommand($"select id, ts from {TableName} order by id")) + { + s_logger.Debug(select.CommandText); + var reader = select.ExecuteReader(); + while (reader.Read()) + { + ++row; + string faultMessage = $"Mismatch for row: {row}, {testCase}"; + Assert.AreEqual(row, reader.GetInt32(0)); + expected.AssertEqual(reader.GetValue(1), comparisonFormat, faultMessage); + } + } + Assert.AreEqual(1+smallBatchRowCount+bigBatchRowCount, row); + } + } + + private void InsertSingleRecord(IDbConnection conn, string sqlInsert, DbType binding, int identifier, ExpectedTimestampWrapper ts) + { + using (var insert = conn.CreateCommand(sqlInsert)) + { + // Arrange + insert.Add("1", DbType.Int32, identifier); + if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType())) + { + var parameter = (SnowflakeDbParameter)insert.Add("2", binding, ts.GetDateTimeOffset()); + parameter.SFDataType = ts.ExpectedColumnType(); + } + else + { + insert.Add("2", binding, ts.GetDateTime()); + } + + // Act + s_logger.Info(sqlInsert); + var rowsAffected = insert.ExecuteNonQuery(); + + // Assert + Assert.AreEqual(1, rowsAffected); + Assert.IsNull(((SnowflakeDbCommand)insert).GetBindStage()); + } + } + + private void InsertMultipleRecords(IDbConnection conn, string sqlInsert, DbType binding, int initialIdentifier, ExpectedTimestampWrapper ts, int rowsCount, bool shouldUseBinding) + { + using (var insert = conn.CreateCommand(sqlInsert)) + { + // Arrange + insert.Add("1", DbType.Int32, Enumerable.Range(initialIdentifier, rowsCount).ToArray()); + if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType())) + { + var parameter = (SnowflakeDbParameter)insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTimeOffset(), rowsCount).ToArray()); + parameter.SFDataType = ts.ExpectedColumnType(); + } + else + { + insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTime(), rowsCount).ToArray()); + } + + // Act + s_logger.Debug(sqlInsert); + var rowsAffected = insert.ExecuteNonQuery(); + + // Assert + Assert.AreEqual(rowsCount, rowsAffected); + if (shouldUseBinding) + Assert.IsNotEmpty(((SnowflakeDbCommand)insert).GetBindStage()); + else + Assert.IsNull(((SnowflakeDbCommand)insert).GetBindStage()); + } + } + + private static string ColumnTypeWithPrecision(SFDataType columnType, Int32? columnPrecision) + => columnPrecision != null ? $"{columnType}({columnPrecision})" : $"{columnType}"; + } + + class ExpectedTimestampWrapper + { + private readonly SFDataType _columnType; + private readonly DateTime? _expectedDateTime; + private readonly DateTimeOffset? _expectedDateTimeOffset; + + internal static ExpectedTimestampWrapper From(string timestampWithTimeZone, SFDataType columnType) + { + if (IsOffsetType(columnType)) + { + var dateTimeOffset = DateTimeOffset.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture); + return new ExpectedTimestampWrapper(dateTimeOffset, columnType); + } + + var dateTime = DateTime.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture); + return new ExpectedTimestampWrapper(dateTime, columnType); + } + + private ExpectedTimestampWrapper(DateTime dateTime, SFDataType columnType) + { + _expectedDateTime = dateTime; + _expectedDateTimeOffset = null; + _columnType = columnType; + } + + private ExpectedTimestampWrapper(DateTimeOffset dateTimeOffset, SFDataType columnType) + { + _expectedDateTimeOffset = dateTimeOffset; + _expectedDateTime = null; + _columnType = columnType; + } + + internal SFDataType ExpectedColumnType() => _columnType; + + internal void AssertEqual(object actual, string comparisonFormat, string faultMessage) + { + switch (_columnType) + { + case SFDataType.TIMESTAMP_TZ: + Assert.AreEqual(GetDateTimeOffset().ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage); + break; + case SFDataType.TIMESTAMP_LTZ: + Assert.AreEqual(GetDateTimeOffset().ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage); + break; + default: + Assert.AreEqual(GetDateTime().ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage); + break; + } + } + + internal DateTime GetDateTime() => _expectedDateTime ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}"); + + internal DateTimeOffset GetDateTimeOffset() => _expectedDateTimeOffset ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}"); + + internal static bool IsOffsetType(SFDataType type) => type == SFDataType.TIMESTAMP_LTZ || type == SFDataType.TIMESTAMP_TZ; } } diff --git a/Snowflake.Data.Tests/SFBaseTest.cs b/Snowflake.Data.Tests/SFBaseTest.cs index 01ae94501..0bb2e1555 100755 --- a/Snowflake.Data.Tests/SFBaseTest.cs +++ b/Snowflake.Data.Tests/SFBaseTest.cs @@ -12,6 +12,7 @@ using System.Runtime.InteropServices; using NUnit.Framework; using Snowflake.Data.Client; +using Snowflake.Data.Log; using Snowflake.Data.Tests.Util; [assembly:LevelOfParallelism(10)] @@ -56,6 +57,8 @@ public static void TearDownContext() #endif public class SFBaseTestAsync { + private static readonly SFLogger s_logger = SFLoggerFactory.GetLogger(); + private const string ConnectionStringWithoutAuthFmt = "scheme={0};host={1};port={2};" + "account={3};role={4};db={5};schema={6};warehouse={7}"; private const string ConnectionStringSnowflakeAuthFmt = ";user={0};password={1};"; @@ -106,10 +109,16 @@ private void RemoveTables() } protected void CreateOrReplaceTable(IDbConnection conn, string tableName, IEnumerable columns, string additionalQueryStr = null) + { + CreateOrReplaceTable(conn, tableName, "", columns, additionalQueryStr); + } + + protected void CreateOrReplaceTable(IDbConnection conn, string tableName, string tableType, IEnumerable columns, string additionalQueryStr = null) { var columnsStr = string.Join(", ", columns); var cmd = conn.CreateCommand(); - cmd.CommandText = $"CREATE OR REPLACE TABLE {tableName}({columnsStr}) {additionalQueryStr}"; + cmd.CommandText = $"CREATE OR REPLACE {tableType} TABLE {tableName}({columnsStr}) {additionalQueryStr}"; + s_logger.Debug(cmd.CommandText); cmd.ExecuteNonQuery(); _tablesToRemove.Add(tableName); diff --git a/Snowflake.Data.Tests/UnitTests/SFBindUploaderTest.cs b/Snowflake.Data.Tests/UnitTests/SFBindUploaderTest.cs new file mode 100644 index 000000000..ac5172086 --- /dev/null +++ b/Snowflake.Data.Tests/UnitTests/SFBindUploaderTest.cs @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. + */ + +using System; +using NUnit.Framework; +using Snowflake.Data.Core; + +namespace Snowflake.Data.Tests.UnitTests +{ + [TestFixture] + class SFBindUploaderTest + { + private readonly SFBindUploader _bindUploader = new SFBindUploader(null, "test"); + + [TestCase(SFDataType.DATE, "0", "1/1/1970")] + [TestCase(SFDataType.DATE, "73785600000", "5/4/1972")] + [TestCase(SFDataType.DATE, "1709164800000", "2/29/2024")] + public void TestCsvDataConversionForDate(SFDataType dbType, string input, string expected) + { + // Arrange + var dateExpected = DateTime.Parse(expected); + var check = SFDataConverter.csharpValToSfVal(SFDataType.DATE, dateExpected); + Assert.AreEqual(check, input); + // Act + DateTime dateActual = DateTime.Parse(_bindUploader.GetCSVData(dbType.ToString(), input)); + // Assert + Assert.AreEqual(dateExpected, dateActual); + } + + [TestCase(SFDataType.TIME, "0", "00:00:00.000000")] + [TestCase(SFDataType.TIME, "100000000", "00:00:00.100000")] + [TestCase(SFDataType.TIME, "1000000000", "00:00:01.000000")] + [TestCase(SFDataType.TIME, "60123456000", "00:01:00.123456")] + [TestCase(SFDataType.TIME, "46801000000000", "13:00:01.000000")] + public void TestCsvDataConversionForTime(SFDataType dbType, string input, string expected) + { + // Arrange + DateTime timeExpected = DateTime.Parse(expected); + var check = SFDataConverter.csharpValToSfVal(SFDataType.TIME, timeExpected); + Assert.AreEqual(check, input); + // Act + DateTime timeActual = DateTime.Parse(_bindUploader.GetCSVData(dbType.ToString(), input)); + // Assert + Assert.AreEqual(timeExpected, timeActual); + } + + [TestCase(SFDataType.TIMESTAMP_LTZ, "39600000000000", "1970-01-01T12:00:00.0000000+01:00")] + [TestCase(SFDataType.TIMESTAMP_LTZ, "1341136800000000000", "2012-07-01T12:00:00.0000000+02:00")] + [TestCase(SFDataType.TIMESTAMP_LTZ, "352245599987654000", "1981-02-28T23:59:59.9876540+02:00")] + [TestCase(SFDataType.TIMESTAMP_LTZ, "1678868249207000000", "2023/03/15T13:17:29.207+05:00")] + public void TestCsvDataConversionForTimestampLtz(SFDataType dbType, string input, string expected) + { + // Arrange + var timestampExpected = DateTimeOffset.Parse(expected); + var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_LTZ, timestampExpected); + Assert.AreEqual(check, input); + // Act + var timestampActual = DateTimeOffset.Parse(_bindUploader.GetCSVData(dbType.ToString(), input)); + // Assert + Assert.AreEqual(timestampExpected.ToLocalTime(), timestampActual); + } + + [TestCase(SFDataType.TIMESTAMP_TZ, "1341136800000000000 1560", "2012-07-01 12:00:00.000000 +02:00")] + [TestCase(SFDataType.TIMESTAMP_TZ, "352245599987654000 1560", "1981-02-28 23:59:59.987654 +02:00")] + public void TestCsvDataConversionForTimestampTz(SFDataType dbType, string input, string expected) + { + // Arrange + DateTimeOffset timestampExpected = DateTimeOffset.Parse(expected); + var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_TZ, timestampExpected); + Assert.AreEqual(check, input); + // Act + DateTimeOffset timestampActual = DateTimeOffset.Parse(_bindUploader.GetCSVData(dbType.ToString(), input)); + // Assert + Assert.AreEqual(timestampExpected, timestampActual); + } + + [TestCase(SFDataType.TIMESTAMP_NTZ, "1341144000000000000", "2012-07-01 12:00:00.000000")] + [TestCase(SFDataType.TIMESTAMP_NTZ, "352252799987654000", "1981-02-28 23:59:59.987654")] + public void TestCsvDataConversionForTimestampNtz(SFDataType dbType, string input, string expected) + { + // Arrange + DateTime timestampExpected = DateTime.Parse(expected); + var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_NTZ, timestampExpected); + Assert.AreEqual(check, input); + // Act + DateTime timestampActual = DateTime.Parse(_bindUploader.GetCSVData(dbType.ToString(), input)); + // Assert + Assert.AreEqual(timestampExpected, timestampActual); + } + + [TestCase(SFDataType.TEXT, "", "\"\"")] + [TestCase(SFDataType.TEXT, "\"", "\"\"\"\"")] + [TestCase(SFDataType.TEXT, "\n", "\"\n\"")] + [TestCase(SFDataType.TEXT, "\t", "\"\t\"")] + [TestCase(SFDataType.TEXT, ",", "\",\"")] + [TestCase(SFDataType.TEXT, "Sample text", "Sample text")] + public void TestCsvDataConversionForText(SFDataType dbType, string input, string expected) + { + // Act + var actual = _bindUploader.GetCSVData(dbType.ToString(), input); + // Assert + Assert.AreEqual(expected, actual); + } + + } +} diff --git a/Snowflake.Data.Tests/Util/DbCommandExtensions.cs b/Snowflake.Data.Tests/Util/DbCommandExtensions.cs new file mode 100644 index 000000000..fb336d5c3 --- /dev/null +++ b/Snowflake.Data.Tests/Util/DbCommandExtensions.cs @@ -0,0 +1,18 @@ +using System.Data; + +namespace Snowflake.Data.Tests.Util +{ + public static class DbCommandExtensions + { + internal static IDbDataParameter Add(this IDbCommand command, string name, DbType dbType, object value) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = name; + parameter.DbType = dbType; + parameter.Value = value; + command.Parameters.Add(parameter); + return parameter; + } + + } +} diff --git a/Snowflake.Data.Tests/Util/DbConnectionExtensions.cs b/Snowflake.Data.Tests/Util/DbConnectionExtensions.cs new file mode 100644 index 000000000..02b7e47dd --- /dev/null +++ b/Snowflake.Data.Tests/Util/DbConnectionExtensions.cs @@ -0,0 +1,23 @@ +using System.Data; + +namespace Snowflake.Data.Tests.Util +{ + public static class DbConnectionExtensions + { + internal static IDbCommand CreateCommand(this IDbConnection connection, string commandText) + { + var command = connection.CreateCommand(); + command.Connection = connection; + command.CommandText = commandText; + return command; + } + + internal static int ExecuteNonQuery(this IDbConnection connection, string commandText) + { + var command = connection.CreateCommand(); + command.Connection = connection; + command.CommandText = commandText; + return command.ExecuteNonQuery(); + } + } +} diff --git a/Snowflake.Data.Tests/Util/TableTypeExtensions.cs b/Snowflake.Data.Tests/Util/TableTypeExtensions.cs new file mode 100644 index 000000000..4c00f3a1d --- /dev/null +++ b/Snowflake.Data.Tests/Util/TableTypeExtensions.cs @@ -0,0 +1,30 @@ +using System; +using NUnit.Framework; + +namespace Snowflake.Data.Tests.Util +{ + public enum SFTableType + { + Standard, + Hybrid, + Iceberg + } + + static class TableTypeExtensions + { + internal static string TableDDLCreationPrefix(this SFTableType val) => val == SFTableType.Standard ? "" : val.ToString().ToUpper(); + + internal static string TableDDLCreationFlags(this SFTableType val) + { + if (val != SFTableType.Iceberg) + return ""; + var externalVolume = Environment.GetEnvironmentVariable("ICEBERG_EXTERNAL_VOLUME"); + var catalog = Environment.GetEnvironmentVariable("ICEBERG_CATALOG"); + var baseLocation = Environment.GetEnvironmentVariable("ICEBERG_BASE_LOCATION"); + Assert.IsNotNull(externalVolume, "env ICEBERG_EXTERNAL_VOLUME not set!"); + Assert.IsNotNull(catalog, "env ICEBERG_CATALOG not set!"); + Assert.IsNotNull(baseLocation, "env ICEBERG_BASE_LOCATION not set!"); + return $"EXTERNAL_VOLUME = '{externalVolume}' CATALOG = '{catalog}' BASE_LOCATION = '{baseLocation}'"; + } + } +} diff --git a/Snowflake.Data/Client/SnowflakeDbCommand.cs b/Snowflake.Data/Client/SnowflakeDbCommand.cs index ce004df5c..36a04f151 100755 --- a/Snowflake.Data/Client/SnowflakeDbCommand.cs +++ b/Snowflake.Data/Client/SnowflakeDbCommand.cs @@ -460,5 +460,7 @@ private void CheckIfCommandTextIsSet() throw new Exception(errorMessage); } } + + internal string GetBindStage() => sfStatement?.GetBindStage(); } } diff --git a/Snowflake.Data/Core/SFBindUploader.cs b/Snowflake.Data/Core/SFBindUploader.cs index a1b3f161d..71dec60fb 100644 --- a/Snowflake.Data/Core/SFBindUploader.cs +++ b/Snowflake.Data/Core/SFBindUploader.cs @@ -224,13 +224,13 @@ internal async Task UploadStreamAsync(MemoryStream stream, string destFileName, statement.SetUploadStream(stream, destFileName, stagePath); await statement.ExecuteTransferAsync(putStmt, cancellationToken).ConfigureAwait(false); } - private string GetCSVData(string sType, string sValue) + + internal string GetCSVData(string sType, string sValue) { if (sValue == null) return sValue; - DateTime dateTime = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Unspecified); - DateTimeOffset dateTimeOffset = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Unspecified); + DateTime epoch = SFDataConverter.UnixEpoch; switch (sType) { case "TEXT": @@ -246,33 +246,29 @@ private string GetCSVData(string sType, string sValue) return '"' + sValue.Replace("\"", "\"\"") + '"'; return sValue; case "DATE": - long dateLong = long.Parse(sValue); - DateTime date = dateTime.AddMilliseconds(dateLong).ToUniversalTime(); + long msFromEpoch = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ms] from Epoch + DateTime date = epoch.AddMilliseconds(msFromEpoch); return date.ToShortDateString(); case "TIME": - long timeLong = long.Parse(sValue); - DateTime time = dateTime.AddMilliseconds(timeLong).ToUniversalTime(); - return time.ToLongTimeString(); + long nsSinceMidnight = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ns] from Midnight + DateTime time = epoch.AddTicks(nsSinceMidnight/100); + return time.ToString("HH:mm:ss.fffffff"); case "TIMESTAMP_LTZ": - long ltzLong = long.Parse(sValue); - TimeSpan ltzts = new TimeSpan(ltzLong / 100); - DateTime ltzdt = dateTime + ltzts; - return ltzdt.ToString(); + long nsFromEpochLtz = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ns] from Epoch + DateTime ltz = epoch.AddTicks(nsFromEpochLtz/100); + return ltz.ToLocalTime().ToString("O"); // ISO 8601 format case "TIMESTAMP_NTZ": - long ntzLong = long.Parse(sValue); - TimeSpan ts = new TimeSpan(ntzLong/100); - DateTime dt = dateTime + ts; - return dt.ToString("yyyy-MM-dd HH:mm:ss.fffffff"); + long nsFromEpochNtz = long.Parse(sValue); // SFDateConverter.csharpValToSfVal provides in [ns] from Epoch + DateTime ntz = epoch.AddTicks(nsFromEpochNtz/100); + return ntz.ToString("yyyy-MM-dd HH:mm:ss.fffffff"); case "TIMESTAMP_TZ": string[] tstzString = sValue.Split(' '); - long tzLong = long.Parse(tstzString[0]); - int tzInt = (int.Parse(tstzString[1]) - 1440) / 60; - TimeSpan tzts = new TimeSpan(tzLong/100); - DateTime tzdt = dateTime + tzts; - TimeSpan tz = new TimeSpan(tzInt, 0, 0); - DateTimeOffset tzDateTimeOffset = new DateTimeOffset(tzdt, tz); + long nsFromEpochTz = long.Parse(tstzString[0]); // SFDateConverter provides in [ns] from Epoch + int timeZoneOffset = int.Parse(tstzString[1]) - 1440; // SFDateConverter provides in minutes increased by 1440m + DateTime timestamp = epoch.AddTicks(nsFromEpochTz/100).AddMinutes(timeZoneOffset); + TimeSpan offset = TimeSpan.FromMinutes(timeZoneOffset); + DateTimeOffset tzDateTimeOffset = new DateTimeOffset(timestamp.Ticks, offset); return tzDateTimeOffset.ToString("yyyy-MM-dd HH:mm:ss.fffffff zzz"); - } return sValue; } diff --git a/Snowflake.Data/Core/SFStatement.cs b/Snowflake.Data/Core/SFStatement.cs index 9252af40e..05e905263 100644 --- a/Snowflake.Data/Core/SFStatement.cs +++ b/Snowflake.Data/Core/SFStatement.cs @@ -147,6 +147,8 @@ internal SFStatement(SFSession session) _restRequester = session.restRequester; } + internal string GetBindStage() => _bindStage; + private void AssignQueryRequestId() { lock (_requestIdLock)