diff --git a/Snowflake.Data.Tests/IntegrationTests/SFConnectionIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFConnectionIT.cs index 2ac74971c..ee1442b56 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFConnectionIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFConnectionIT.cs @@ -372,7 +372,7 @@ public void TestConnectViaSecureString() } } - [Test, NonParallelizable] + [Test] public void TestLoginTimeout() { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -410,7 +410,7 @@ public void TestLoginTimeout() } } - [Test, NonParallelizable] + [Test] public void TestLoginWithMaxRetryReached() { using (IDbConnection conn = new MockSnowflakeDbConnection()) @@ -438,7 +438,7 @@ public void TestLoginWithMaxRetryReached() } } - [Test, NonParallelizable] + [Test] [Ignore("Disable unstable test cases for now")] public void TestDefaultLoginTimeout() { @@ -1660,7 +1660,7 @@ class SFConnectionITAsync : SFBaseTestAsync private static SFLogger logger = SFLoggerFactory.GetLogger(); - [Test, NonParallelizable] + [Test] public void TestCancelLoginBeforeTimeout() { using (var conn = new MockSnowflakeDbConnection()) @@ -1708,7 +1708,7 @@ public void TestCancelLoginBeforeTimeout() } } - [Test, NonParallelizable] + [Test] public void TestAsyncLoginTimeout() { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -1748,7 +1748,7 @@ public void TestAsyncLoginTimeout() } } - [Test, NonParallelizable] + [Test] public void TestAsyncDefaultLoginTimeout() { using (var conn = new MockSnowflakeDbConnection()) @@ -1884,7 +1884,7 @@ public void TestCloseAsync() } #endif - [Test, NonParallelizable] + [Test] public void TestCloseAsyncFailure() { using (var conn = new MockSnowflakeDbConnection(new MockCloseSessionException())) diff --git a/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs index 8532b3bb0..111094b74 100755 --- a/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs @@ -227,8 +227,7 @@ public void TestSimpleLargeResultSet() conn.Close(); } } - - + [Test, NonParallelizable] public void TestUseV1ResultParser() { diff --git a/Snowflake.Data.Tests/IntegrationTests/SFDbDataReaderIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFDbDataReaderIT.cs index 158abc7f0..3b55ff682 100755 --- a/Snowflake.Data.Tests/IntegrationTests/SFDbDataReaderIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFDbDataReaderIT.cs @@ -8,26 +8,37 @@ using System.Data; using System.Globalization; using System.Text; +using NUnit.Framework; +using Snowflake.Data.Client; +using Snowflake.Data.Core; +using Snowflake.Data.Tests.Util; namespace Snowflake.Data.Tests.IntegrationTests { - using Snowflake.Data.Client; - using Snowflake.Data.Core; - using NUnit.Framework; - - [TestFixture] + // TODO: enable tests for Arrow + //[TestFixture(ResultFormat.ARROW)] + [TestFixture(ResultFormat.JSON)] class SFDbDataReaderIT : SFBaseTest { - static private readonly Random rand = new Random(); + protected override string TestName => base.TestName + _resultFormat; + + private readonly ResultFormat _resultFormat; + + public SFDbDataReaderIT(ResultFormat resultFormat) + { + _resultFormat = resultFormat; + } + + private void ValidateResultFormat(IDataReader reader) + { + Assert.AreEqual(_resultFormat, ((SnowflakeDbDataReader)reader).ResultFormat); + } [Test] - public void testRecordsAffected() + public void TestRecordsAffected() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new []{"cola NUMBER"}); IDbCommand cmd = conn.CreateCommand(); @@ -46,19 +57,16 @@ public void testRecordsAffected() Assert.AreEqual(0, count); // Reader's RecordsAffected should be available even if the connection is closed - conn.Close(); + CloseConnection(conn); Assert.AreEqual(3, reader.RecordsAffected); } } [Test] - public void testGetNumber() + public void TestGetNumber() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new []{"cola NUMBER"}); IDbCommand cmd = conn.CreateCommand(); @@ -93,7 +101,9 @@ public void testGetNumber() cmd.CommandText = $"select * from {TableName}"; IDataReader reader = cmd.ExecuteReader(); - + + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); Assert.AreEqual(numInt, reader.GetInt32(0)); @@ -106,19 +116,16 @@ public void testGetNumber() Assert.IsFalse(reader.Read()); reader.Close(); - conn.Close(); + CloseConnection(conn); } } [Test] - public void testGetDouble() + public void TestGetDouble() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new []{"cola DOUBLE"}); IDbCommand cmd = conn.CreateCommand(); @@ -147,6 +154,8 @@ public void testGetDouble() cmd.CommandText = $"select * from {TableName}"; IDataReader reader = cmd.ExecuteReader(); + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); Assert.AreEqual(numFloat, reader.GetFloat(0)); Assert.AreEqual((decimal)numFloat, reader.GetDecimal(0)); @@ -158,7 +167,7 @@ public void testGetDouble() Assert.IsFalse(reader.Read()); reader.Close(); - conn.Close(); + CloseConnection(conn); } } @@ -171,7 +180,7 @@ public void testGetDouble() [TestCase("1900-09-03 00:00:00.0000000")] public void TestGetDate(string inputTimeStr) { - testGetDateAndOrTime(inputTimeStr, null, SFDataType.DATE); + TestGetDateAndOrTime(inputTimeStr, null, SFDataType.DATE); } @@ -186,9 +195,9 @@ public void TestGetDate(string inputTimeStr) [TestCase("1969-07-21 02:56:15.1234567", 1)] [TestCase("1900-09-03 12:12:12.1212121", null)] [TestCase("1900-09-03 12:12:12.1212121", 1)] - public void testGetTime(string inputTimeStr, int? precision) + public void TestGetTime(string inputTimeStr, int? precision) { - testGetDateAndOrTime(inputTimeStr, precision, SFDataType.TIME); + TestGetDateAndOrTime(inputTimeStr, precision, SFDataType.TIME); } [Test] @@ -206,13 +215,10 @@ public void testGetTime(string inputTimeStr, int? precision) [TestCase("23:59:59.1234567")] [TestCase("23:59:59.12345678")] [TestCase("23:59:59.123456789")] - public void testGetTimeSpan(string inputTimeStr) + public void TestGetTimeSpan(string inputTimeStr) { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - // Insert data int fractionalPartIndex = inputTimeStr.IndexOf('.'); var precision = fractionalPartIndex > 0 ? inputTimeStr.Length - (inputTimeStr.IndexOf('.') + 1) : 0; @@ -230,6 +236,8 @@ public void testGetTimeSpan(string inputTimeStr) cmd.CommandText = $"SELECT cola FROM {TableName}"; IDataReader reader = cmd.ExecuteReader(); + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); // For time, we getDateTime on the column and ignore date part @@ -246,19 +254,16 @@ public void testGetTimeSpan(string inputTimeStr) Assert.AreEqual(dateTimeTime.Second, timeSpanTime.Seconds); Assert.AreEqual(dateTimeTime.Millisecond, timeSpanTime.Milliseconds); - conn.Close(); + CloseConnection(conn); } } [Test] - public void testGetTimeSpanError() + public void TestGetTimeSpanError() { // Only Time data can be retrieved using GetTimeSpan, other type will fail - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new [] { "C1 NUMBER", @@ -299,6 +304,8 @@ public void testGetTimeSpanError() cmd.CommandText = $"SELECT * FROM {TableName}"; IDataReader reader = cmd.ExecuteReader(); + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); // All types except TIME fail conversion when calling GetTimeSpan @@ -334,11 +341,11 @@ public void testGetTimeSpanError() reader.Close(); - conn.Close(); + CloseConnection(conn); } } - private void testGetDateAndOrTime(string inputTimeStr, int? precision, SFDataType dataType) + private void TestGetDateAndOrTime(string inputTimeStr, int? precision, SFDataType dataType) { // Can't use DateTime object as test case, must parse. DateTime inputTime; @@ -351,11 +358,8 @@ private void testGetDateAndOrTime(string inputTimeStr, int? precision, SFDataTyp inputTime = DateTime.ParseExact(inputTimeStr, "yyyy-MM-dd HH:mm:ss.fffffff", CultureInfo.InvariantCulture); } - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new [] { $"cola {dataType}{ (precision == null ? string.Empty : $"({precision})" )}" @@ -391,6 +395,8 @@ private void testGetDateAndOrTime(string inputTimeStr, int? precision, SFDataTyp cmd.CommandText = $"select * from {TableName}"; IDataReader reader = cmd.ExecuteReader(); + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); // For time, we getDateTime on the column and ignore date part @@ -422,7 +428,7 @@ private void testGetDateAndOrTime(string inputTimeStr, int? precision, SFDataTyp reader.Close(); - conn.Close(); + CloseConnection(conn); } } @@ -441,23 +447,25 @@ private void testGetDateAndOrTime(string inputTimeStr, int? precision, SFDataTyp [TestCase("1969-07-21 02:56:15.0000000", 1)] //dates w/o second fractions before the unix epoch are fine //[TestCase("1900-09-03 12:12:12.1212121", null)] // fails [TestCase("1900-09-03 12:12:12.0000000", 1)] - public void testGetTimestampNTZ(string inputTimeStr, int? precision) + public void TestGetTimestampNTZ(string inputTimeStr, int? precision) { - testGetDateAndOrTime(inputTimeStr, precision, SFDataType.TIMESTAMP_NTZ); + TestGetDateAndOrTime(inputTimeStr, precision, SFDataType.TIMESTAMP_NTZ); } [Test] - public void testGetTimestampTZ() + [TestCase(0)] + [TestCase(5)] + [TestCase(-5)] + [TestCase(14)] + [TestCase(-14)] + public void TestGetTimestampTZ(int timezoneOffsetInHours) { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new []{"cola TIMESTAMP_TZ"}); - DateTimeOffset now = DateTimeOffset.Now; + DateTimeOffset now = DateTimeOffset.Now.ToOffset(TimeSpan.FromHours(timezoneOffsetInHours)); IDbCommand cmd = conn.CreateCommand(); @@ -476,25 +484,25 @@ public void testGetTimestampTZ() cmd.CommandText = $"select * from {TableName}"; IDataReader reader = cmd.ExecuteReader(); + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); DateTimeOffset dtOffset = (DateTimeOffset)reader.GetValue(0); reader.Close(); - Assert.AreEqual(0, DateTimeOffset.Compare(now, dtOffset)); + Assert.AreEqual(now, dtOffset); + Assert.AreEqual(now.Offset, dtOffset.Offset); - conn.Close(); + CloseConnection(conn); } } [Test] - public void testGetTimestampLTZ() + public void TestGetTimestampLTZ() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new []{"cola TIMESTAMP_LTZ"}); DateTimeOffset now = DateTimeOffset.Now; @@ -517,24 +525,24 @@ public void testGetTimestampLTZ() cmd.CommandText = $"select * from {TableName}"; IDataReader reader = cmd.ExecuteReader(); + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); DateTimeOffset dtOffset = (DateTimeOffset)reader.GetValue(0); reader.Close(); - Assert.AreEqual(0, DateTimeOffset.Compare(now, dtOffset)); + Assert.AreEqual(now, dtOffset); + Assert.AreEqual(now.Offset, dtOffset.Offset); - conn.Close(); + CloseConnection(conn); } } [Test] - public void testGetBoolean() + public void TestGetBoolean() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new []{"cola BOOLEAN"}); IDbCommand cmd = conn.CreateCommand(); @@ -554,22 +562,21 @@ public void testGetBoolean() cmd.CommandText = $"select * from {TableName}"; IDataReader reader = cmd.ExecuteReader(); + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); Assert.IsTrue(reader.GetBoolean(0)); reader.Close(); - conn.Close(); + CloseConnection(conn); } } [Test] - public void testGetBinary() + public void TestGetBinary() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new [] { "col1 BINARY", @@ -596,6 +603,8 @@ public void testGetBinary() cmd.CommandText = $"select * from {TableName}"; IDataReader reader = cmd.ExecuteReader(); + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); // Auto type conversion Assert.IsTrue(testBytes.SequenceEqual((byte[])reader.GetValue(0))); @@ -701,18 +710,15 @@ public void testGetBinary() reader.Close(); - conn.Close(); + CloseConnection(conn); } } [Test] - public void testGetChars() + public void TestGetChars() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new [] { "col1 VARCHAR(50)", @@ -739,6 +745,8 @@ public void testGetChars() cmd.CommandText = $"select * from {TableName}"; IDataReader reader = cmd.ExecuteReader(); + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); // Auto type conversion Assert.IsTrue(testChars.Equals(reader.GetValue(0))); @@ -846,18 +854,15 @@ public void testGetChars() reader.Close(); - conn.Close(); + CloseConnection(conn); } } [Test] - public void testGetStream() + public void TestGetStream() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new [] { "col1 VARCHAR(50)", @@ -884,6 +889,8 @@ public void testGetStream() cmd.CommandText = $"select * from {TableName}"; DbDataReader reader = (DbDataReader) cmd.ExecuteReader(); + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); // Auto type conversion @@ -920,23 +927,22 @@ public void testGetStream() reader.Close(); - conn.Close(); + CloseConnection(conn); } } [Test] - public void testGetValueIndexOutOfBound() + public void TestGetValueIndexOutOfBound() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - IDbCommand cmd = conn.CreateCommand(); cmd.CommandText = "select 1"; IDataReader reader = cmd.ExecuteReader(); + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); try @@ -960,23 +966,22 @@ public void testGetValueIndexOutOfBound() } reader.Close(); - conn.Close(); + CloseConnection(conn); } } [Test] - public void testBasicDataReader() + public void TestBasicDataReader() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - using (IDbCommand cmd = conn.CreateCommand()) { cmd.CommandText = "select 1 as colone, 2 as coltwo"; using (IDataReader reader = cmd.ExecuteReader()) { + ValidateResultFormat(reader); + Assert.AreEqual(2, reader.FieldCount); Assert.AreEqual(0, reader.Depth); Assert.IsTrue(((SnowflakeDbDataReader)reader).HasRows); @@ -1022,18 +1027,15 @@ public void testBasicDataReader() } } - conn.Close(); + CloseConnection(conn); } } [Test] - public void testReadOutNullVal() + public void TestReadOutNullVal() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new [] { "a INTEGER", @@ -1049,6 +1051,8 @@ public void testReadOutNullVal() cmd.CommandText = $"select * from {TableName}"; using (IDataReader reader = cmd.ExecuteReader()) { + ValidateResultFormat(reader); + reader.Read(); object nullVal = reader.GetValue(0); Assert.AreEqual(DBNull.Value, nullVal); @@ -1059,18 +1063,15 @@ public void testReadOutNullVal() } } - conn.Close(); + CloseConnection(conn); } } [Test] - public void testGetGuid() + public void TestGetGuid() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new []{"cola STRING"}); IDbCommand cmd = conn.CreateCommand(); @@ -1091,6 +1092,8 @@ public void testGetGuid() cmd.CommandText = $"select * from {TableName}"; IDataReader reader = cmd.ExecuteReader(); + ValidateResultFormat(reader); + Assert.IsTrue(reader.Read()); Assert.AreEqual(val, reader.GetGuid(0)); @@ -1104,20 +1107,16 @@ public void testGetGuid() reader.Close(); - conn.Close(); + CloseConnection(conn); } - } [Test] public void TestCopyCmdUpdateCount() { var stageName = TestName; - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new []{"cola STRING"}); IDbCommand cmd = conn.CreateCommand(); @@ -1140,7 +1139,7 @@ public void TestCopyCmdUpdateCount() cmd.CommandText = $"drop stage {stageName}"; cmd.ExecuteNonQuery(); - conn.Close(); + CloseConnection(conn); } } @@ -1148,11 +1147,8 @@ public void TestCopyCmdUpdateCount() public void TestCopyCmdResultSet() { var stageName = TestName; - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new []{"cola STRING"}); IDbCommand cmd = conn.CreateCommand(); @@ -1185,18 +1181,15 @@ public void TestCopyCmdResultSet() cmd.CommandText = $"drop stage {stageName}"; cmd.ExecuteNonQuery(); - conn.Close(); + CloseConnection(conn); } } [Test] public void TestRetrieveSemiStructuredData() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new [] { "cola VARIANT", @@ -1210,24 +1203,23 @@ public void TestRetrieveSemiStructuredData() cmd.CommandText = $"select * from {TableName}"; using (IDataReader reader = cmd.ExecuteReader()) { + ValidateResultFormat(reader); + Assert.AreEqual(true, reader.Read()); Assert.AreEqual("[\n \"1\",\n \"2\"\n]", reader.GetString(0)); Assert.AreEqual("[\n \"1\",\n \"2\"\n]", reader.GetString(1)); Assert.AreEqual("{\n \"key\": \"value\"\n}", reader.GetString(2)); } - conn.Close(); + CloseConnection(conn); } } [Test] public void TestResultSetMetadata() { - using (IDbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - CreateOrReplaceTable(conn, TableName, new [] { "c1 NUMBER(20, 4)", @@ -1243,6 +1235,8 @@ public void TestResultSetMetadata() cmd.CommandText = $"select * from {TableName}"; using (IDataReader reader = cmd.ExecuteReader()) { + ValidateResultFormat(reader); + var dataTable = reader.GetSchemaTable(); dataTable.DefaultView.Sort = SchemaTableColumn.ColumnName; dataTable = dataTable.DefaultView.ToTable(); @@ -1289,36 +1283,33 @@ public void TestResultSetMetadata() Assert.AreEqual(true, row[SchemaTableColumn.AllowDBNull]); } - conn.Close(); + CloseConnection(conn); } } [Test] public void TestHasRows() { - using (DbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - DbCommand cmd = conn.CreateCommand(); cmd.CommandText = "select 1 where 1=2"; DbDataReader reader = cmd.ExecuteReader(); + + ValidateResultFormat(reader); + Assert.IsFalse(reader.HasRows); reader.Close(); - conn.Close(); + CloseConnection(conn); } } [Test] public void TestHasRowsMultiStatement() { - using (DbConnection conn = new SnowflakeDbConnection()) + using (var conn = CreateAndOpenConnection()) { - conn.ConnectionString = ConnectionString; - conn.Open(); - DbCommand cmd = conn.CreateCommand(); cmd.CommandText = "select 1;" + "select 1 where 1=2;" + @@ -1333,6 +1324,9 @@ public void TestHasRowsMultiStatement() DbDataReader reader = cmd.ExecuteReader(); + // multi statements are always returned in JSON + Assert.AreEqual(ResultFormat.JSON, ((SnowflakeDbDataReader)reader).ResultFormat); + // select 1 Assert.IsTrue(reader.HasRows); reader.Read(); @@ -1355,8 +1349,156 @@ public void TestHasRowsMultiStatement() Assert.IsFalse(reader.HasRows); reader.Close(); - conn.Close(); + CloseConnection(conn); } } + + [Test] + [TestCase("99")] // Int8 + [TestCase("9.9")] // Int8 + scale + [TestCase("999")] // Int16 + [TestCase("9.99")] // Int16 + scale + [TestCase("99999999")] // Int32 + [TestCase("999999.99")] // Int32 + scale + [TestCase("99999999999")] // Int64 + [TestCase("999999999.99")] // Int64 + scale + [TestCase("999999999999999999999999999")] // Decimal + [TestCase("9999999999999999999999999.99")] // Decimal + scale + public void TestNumericValues(string testValue) + { + using (var conn = CreateAndOpenConnection()) + { + DbCommand cmd = conn.CreateCommand(); + cmd.CommandText = "select " + testValue; + using (SnowflakeDbDataReader reader = (SnowflakeDbDataReader)cmd.ExecuteReader()) + { + ValidateResultFormat(reader); + + while (reader.Read()) + { + Assert.AreEqual(Convert.ToDecimal(testValue), reader.GetDecimal(0)); + Assert.AreEqual(Convert.ToDouble(testValue), reader.GetDouble(0)); + Assert.AreEqual(Convert.ToSingle(testValue), reader.GetFloat(0)); + if (!testValue.Contains('.')) + { + decimal value = Decimal.Parse(testValue); + if (value >= Int64.MinValue && value <= Int64.MaxValue) + Assert.AreEqual(Convert.ToInt64(testValue), reader.GetInt64(0)); + else + Assert.Throws(() => reader.GetInt64(0)); + if (value >= Int32.MinValue && value <= Int32.MaxValue) + Assert.AreEqual(Convert.ToInt32(testValue), reader.GetInt32(0)); + else + Assert.Throws(() => reader.GetInt32(0)); + if (value >= Int16.MinValue && value <= Int16.MaxValue) + Assert.AreEqual(Convert.ToInt16(testValue), reader.GetInt16(0)); + else + Assert.Throws(() => reader.GetInt16(0)); + if (value >= 0 && value <= 255) + Assert.AreEqual(Convert.ToByte(testValue), reader.GetByte(0)); + else + Assert.Throws(() => reader.GetByte(0)); + } + } + CloseConnection(conn); + } + } + } + + [Test] + [TestCase("2019-01-01 12:12:12.1234567 +0500", 7)] + [TestCase("2019-01-01 12:12:12.1234567 -0500", 7)] + [TestCase("2019-01-01 12:12:12.1234567 +1400", 7)] + [TestCase("2019-01-01 12:12:12.1234567 -1400", 7)] + [TestCase("0001-01-01 00:00:00.0000000 +0000", 9)] + [TestCase("9999-12-31 23:59:59.9999999 +0000", 9)] + public void TestTimestampTz(string testValue, int scale) + { + using (var conn = CreateAndOpenConnection()) + { + DbCommand cmd = conn.CreateCommand(); + + cmd.CommandText = $"select '{testValue}'::TIMESTAMP_TZ({scale})"; + using (SnowflakeDbDataReader reader = (SnowflakeDbDataReader)cmd.ExecuteReader()) + { + ValidateResultFormat(reader); + + reader.Read(); + + var expectedValue = DateTimeOffset.Parse(testValue); + + Assert.AreEqual(expectedValue, reader.GetValue(0)); + } + + CloseConnection(conn); + } + } + + [Test] + [TestCase("2019-01-01 12:12:12.1234567 +0500", 7)] + [TestCase("2019-01-01 12:12:12.1234567 +1400", 7)] + [TestCase("0001-01-01 00:00:00.0000000 +0000", 9)] + [TestCase("9999-12-31 23:59:59.9999999 +0000", 9)] + public void TestTimestampLtz(string testValue, int scale) + { + using (var conn = CreateAndOpenConnection()) + { + DbCommand cmd = conn.CreateCommand(); + + cmd.CommandText = $"select '{testValue}'::TIMESTAMP_LTZ({scale})"; + using (SnowflakeDbDataReader reader = (SnowflakeDbDataReader)cmd.ExecuteReader()) + { + ValidateResultFormat(reader); + + reader.Read(); + + var expectedValue = DateTimeOffset.Parse(testValue).ToLocalTime(); + + Assert.AreEqual(expectedValue, reader.GetValue(0)); + } + + CloseConnection(conn); + } + } + + [Test] + [TestCase("2019-01-01 12:12:12.1234567", 7)] + [TestCase("0001-01-01 00:00:00.0000000", 9)] + [TestCase("9999-12-31 23:59:59.9999999", 9)] + public void TestTimestampNtz(string testValue, int scale) + { + using (var conn = CreateAndOpenConnection()) + { + DbCommand cmd = conn.CreateCommand(); + + cmd.CommandText = $"select '{testValue}'::TIMESTAMP_NTZ({scale})"; + using (SnowflakeDbDataReader reader = (SnowflakeDbDataReader)cmd.ExecuteReader()) + { + ValidateResultFormat(reader); + + reader.Read(); + + var expectedValue = DateTime.Parse(testValue); + + Assert.AreEqual(expectedValue, reader.GetValue(0)); + } + + CloseConnection(conn); + } + } + + private DbConnection CreateAndOpenConnection() + { + var conn = new SnowflakeDbConnection(ConnectionString); + conn.Open(); + SessionParameterAlterer.SetResultFormat(conn, _resultFormat); + return conn; + } + + private void CloseConnection(DbConnection conn) + { + SessionParameterAlterer.RestoreResultFormat(conn); + conn.Close(); + } } -} \ No newline at end of file +} diff --git a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs index 02b6fbe53..382fabe04 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs @@ -19,6 +19,7 @@ namespace Snowflake.Data.Tests.IntegrationTests using Snowflake.Data.Core.FileTransfer; [TestFixture] + [Parallelizable(ParallelScope.Children)] class SFPutGetTest : SFBaseTest { private const int NumberOfRows = 4; @@ -337,7 +338,6 @@ public void TestPutDirectoryMixedWildcard() } [Test] - [Parallelizable(ParallelScope.All)] public void TestPutGetCommand( [Values("none", "gzip", "bzip2", "brotli", "deflate", "raw_deflate", "zstd")] string sourceFileCompressionType, [Values] StageType stageType, @@ -358,7 +358,6 @@ public void TestPutGetCommand( // Test small file upload/download with GCS_USE_DOWNSCOPED_CREDENTIAL set to true [Test] [IgnoreOnEnvIs("snowflake_cloud_env", new [] { "AWS", "AZURE" })] - [Parallelizable(ParallelScope.All)] public void TestPutGetGcsDownscopedCredential( [Values] StageType stageType, [Values("", "/TEST_PATH")] string stagePath) diff --git a/Snowflake.Data.Tests/IntegrationTests/SFReusableChunkTest.cs b/Snowflake.Data.Tests/IntegrationTests/SFReusableChunkTest.cs index 54c0fe49e..1dcc4ddc2 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFReusableChunkTest.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFReusableChunkTest.cs @@ -1,4 +1,5 @@ using System; +using Snowflake.Data.Tests.Util; namespace Snowflake.Data.Tests.IntegrationTests { @@ -13,13 +14,14 @@ namespace Snowflake.Data.Tests.IntegrationTests class SFReusableChunkTest : SFBaseTest { [Test] - public void testDelCharPr431() + public void TestDelCharPr431() { using (IDbConnection conn = new SnowflakeDbConnection()) { conn.ConnectionString = ConnectionString; conn.Open(); + SessionParameterAlterer.SetResultFormat(conn, ResultFormat.JSON); CreateOrReplaceTable(conn, TableName, new []{"col STRING"}); IDbCommand cmd = conn.CreateCommand(); @@ -50,13 +52,14 @@ public void testDelCharPr431() } Assert.AreEqual(largeTableRowCount, rowCount); + SessionParameterAlterer.RestoreResultFormat(conn); // Reader's RecordsAffected should be available even if the connection is closed conn.Close(); } } [Test] - public void testParseJson() + public void TestParseJson() { IChunkParserFactory previous = ChunkParserFactory.Instance; ChunkParserFactory.Instance = new TestChunkParserFactory(1); @@ -66,6 +69,7 @@ public void testParseJson() conn.ConnectionString = ConnectionString; conn.Open(); + SessionParameterAlterer.SetResultFormat(conn, ResultFormat.JSON); CreateOrReplaceTable(conn, TableName, new []{"src VARIANT"}); IDbCommand cmd = conn.CreateCommand(); @@ -109,6 +113,7 @@ select parse_json('{{ } Assert.AreEqual(500, rowCount); + SessionParameterAlterer.RestoreResultFormat(conn); // Reader's RecordsAffected should be available even if the connection is closed conn.Close(); } @@ -116,7 +121,7 @@ select parse_json('{{ } [Test] - public void testChunkRetry() + public void TestChunkRetry() { IChunkParserFactory previous = ChunkParserFactory.Instance; ChunkParserFactory.Instance = new TestChunkParserFactory(6); // lower than default retry of 7 @@ -126,6 +131,7 @@ public void testChunkRetry() conn.ConnectionString = ConnectionString; conn.Open(); + SessionParameterAlterer.SetResultFormat(conn, ResultFormat.JSON); CreateOrReplaceTable(conn, TableName, new []{"col STRING"}); IDbCommand cmd = conn.CreateCommand(); @@ -156,6 +162,7 @@ public void testChunkRetry() } Assert.AreEqual(largeTableRowCount, rowCount); + SessionParameterAlterer.RestoreResultFormat(conn); // Reader's RecordsAffected should be available even if the connection is closed conn.Close(); } @@ -164,7 +171,7 @@ public void testChunkRetry() } [Test] - public void testExceptionThrownWhenChunkDownloadRetryCountExceeded() + public void TestExceptionThrownWhenChunkDownloadRetryCountExceeded() { IChunkParserFactory previous = ChunkParserFactory.Instance; ChunkParserFactory.Instance = new TestChunkParserFactory(8); // larger than default max retry of 7 diff --git a/Snowflake.Data.Tests/Mock/MockSnowflakeDbConnection.cs b/Snowflake.Data.Tests/Mock/MockSnowflakeDbConnection.cs index 0f1758636..c6d8f0698 100644 --- a/Snowflake.Data.Tests/Mock/MockSnowflakeDbConnection.cs +++ b/Snowflake.Data.Tests/Mock/MockSnowflakeDbConnection.cs @@ -78,7 +78,7 @@ public override Task OpenAsync(CancellationToken cancellationToken) cancellationToken); } - + private void SetMockSession() { SfSession = new SFSession(ConnectionString, Password, _restRequester); diff --git a/Snowflake.Data.Tests/SFBaseTest.cs b/Snowflake.Data.Tests/SFBaseTest.cs index 3f8c9207d..01ae94501 100755 --- a/Snowflake.Data.Tests/SFBaseTest.cs +++ b/Snowflake.Data.Tests/SFBaseTest.cs @@ -59,7 +59,7 @@ public class SFBaseTestAsync private const string ConnectionStringWithoutAuthFmt = "scheme={0};host={1};port={2};" + "account={3};role={4};db={5};schema={6};warehouse={7}"; private const string ConnectionStringSnowflakeAuthFmt = ";user={0};password={1};"; - protected readonly string TestName = TestContext.CurrentContext.Test.MethodName; + protected virtual string TestName => TestContext.CurrentContext.Test.MethodName; protected string TestNameWithWorker => TestName + TestContext.CurrentContext.WorkerId?.Replace("#", "_"); protected string TableName => TestNameWithWorker; @@ -239,7 +239,7 @@ public void CreateTestTimeArtifact() // Snowflake.Data.Tests/bin/debug/{.net_version}/ File.WriteAllText($"..{separator}..{separator}..{separator}{GetOs()}_{dotnetVersion}_{cloudEnv}_performance.csv", resultText); } - + private static string s_connectionString => string.Format(ConnectionStringFmt, TestConfig.protocol, TestConfig.host, diff --git a/Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs b/Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs index 673e95c84..3ea66f0be 100755 --- a/Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs +++ b/Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs @@ -3,10 +3,13 @@ */ using System; +using System.Collections; using System.Linq; using Apache.Arrow; +using Apache.Arrow.Types; using NUnit.Framework; using Snowflake.Data.Core; +using Snowflake.Data.Tests.Util; namespace Snowflake.Data.Tests.UnitTests { @@ -23,71 +26,70 @@ class ArrowResultChunkTest .Append("Col_Int32", false, col => col.Int32( array => array.AppendRange(Enumerable.Range(1, RowCountBatchTwo)))) .Build(); - private ArrowResultChunk _chunk; [Test] public void TestAddRecordBatchAddsBatchTwo() { - _chunk = new ArrowResultChunk(_recordBatchOne); - _chunk.AddRecordBatch(_recordBatchTwo); + var chunk = new ArrowResultChunk(_recordBatchOne); + chunk.AddRecordBatch(_recordBatchTwo); - Assert.AreEqual(2, _chunk.RecordBatch.Count); + Assert.AreEqual(2, chunk.RecordBatch.Count); } [Test] public void TestNextIteratesThroughAllRecordsOfOneBatch() { - _chunk = new ArrowResultChunk(_recordBatchOne); + var chunk = new ArrowResultChunk(_recordBatchOne); for (var i = 0; i < RowCountBatchOne; ++i) { - Assert.IsTrue(_chunk.Next()); + Assert.IsTrue(chunk.Next()); } - Assert.IsFalse(_chunk.Next()); + Assert.IsFalse(chunk.Next()); } [Test] public void TestNextIteratesThroughAllRecordsOfTwoBatches() { - _chunk = new ArrowResultChunk(_recordBatchOne); - _chunk.AddRecordBatch(_recordBatchTwo); + var chunk = new ArrowResultChunk(_recordBatchOne); + chunk.AddRecordBatch(_recordBatchTwo); for (var i = 0; i < RowCountBatchOne + RowCountBatchTwo; ++i) { - Assert.IsTrue(_chunk.Next()); + Assert.IsTrue(chunk.Next()); } - Assert.IsFalse(_chunk.Next()); + Assert.IsFalse(chunk.Next()); } [Test] public void TestRewindIteratesThroughAllRecordsOfBatchOne() { - _chunk = new ArrowResultChunk(_recordBatchOne); + var chunk = new ArrowResultChunk(_recordBatchOne); // move to the end of the batch - while (_chunk.Next()) {} + while (chunk.Next()) {} for (var i = 0; i < RowCountBatchOne; ++i) { - Assert.IsTrue(_chunk.Rewind()); + Assert.IsTrue(chunk.Rewind()); } - Assert.IsFalse(_chunk.Rewind()); + Assert.IsFalse(chunk.Rewind()); } [Test] public void TestRewindIteratesThroughAllRecordsOfTwoBatches() { - _chunk = new ArrowResultChunk(_recordBatchOne); - _chunk.AddRecordBatch(_recordBatchTwo); + var chunk = new ArrowResultChunk(_recordBatchOne); + chunk.AddRecordBatch(_recordBatchTwo); // move to the end of the batch - while (_chunk.Next()) {} + while (chunk.Next()) {} for (var i = 0; i < RowCountBatchOne + RowCountBatchTwo; ++i) { - Assert.IsTrue(_chunk.Rewind()); + Assert.IsTrue(chunk.Rewind()); } - Assert.IsFalse(_chunk.Rewind()); + Assert.IsFalse(chunk.Rewind()); } [Test] @@ -99,69 +101,485 @@ public void TestResetClearsChunkData() uncompressedSize = 100, rowCount = 2 }; - _chunk = new ArrowResultChunk(_recordBatchOne); + var chunk = new ArrowResultChunk(_recordBatchOne); - _chunk.Reset(chunkInfo, 0); + chunk.Reset(chunkInfo, 0); - Assert.AreEqual(0, _chunk.ChunkIndex); - Assert.AreEqual(chunkInfo.url, _chunk.Url); - Assert.AreEqual(chunkInfo.rowCount, _chunk.RowCount); + Assert.AreEqual(0, chunk.ChunkIndex); + Assert.AreEqual(chunkInfo.url, chunk.Url); + Assert.AreEqual(chunkInfo.rowCount, chunk.RowCount); + } + + [Test] + public void TestRowCountReturnsNumberOfRows() + { + var chunk = new ArrowResultChunk(_recordBatchOne); + + Assert.AreEqual(RowCountBatchOne, chunk.RowCount); } [Test] - public void TestExtractCellWithRowParameterReadsAllRows() + public void TestGetChunkIndexReturnsFirstChunk() { - _chunk = new ArrowResultChunk(_recordBatchOne); + var chunk = new ArrowResultChunk(_recordBatchOne); - var column = (Int32Array)_recordBatchOne.Column(0); - for (var i = 0; i < RowCountBatchOne; ++i) + Assert.AreEqual(0, chunk.ChunkIndex); + } + + [Test] + public void TestUnusedExtractCellThrowsNotSupportedException() + { + var chunk = new ArrowResultChunk(_recordBatchOne); + + Assert.Throws(() => chunk.ExtractCell(0)); + Assert.Throws(() => chunk.ExtractCell(0, 0)); + } + + [Test] + public void TestExtractCellReturnsDecimal() + { + var testValues = new decimal[] { 0, 100, -100, Decimal.MaxValue, Decimal.MinValue }; + var sfType = SFDataType.FIXED; + + for (var scale = 0; scale <= 9; ++scale) { - var valueFromRecordBatch = column.GetValue(i).ToString(); - Assert.AreEqual(valueFromRecordBatch, _chunk.ExtractCell(i, 0).SafeToString()); + TestExtractCell(testValues, sfType, scale, (long)Math.Pow(10, scale)); } } [Test] - public void TestExtractCellReadsAllRows() + public void TestExtractCellReturnsNumber64() { - _chunk = new ArrowResultChunk(_recordBatchOne); + var testValues = new long[] { 0, 100, -100, Int64.MaxValue, Int64.MinValue }; + var sfType = SFDataType.FIXED; - var column = (Int32Array)_recordBatchOne.Column(0); - for (var i = 0; i < RowCountBatchOne; ++i) + for (var scale = 0; scale <= 9; ++scale) { - var valueFromRecordBatch = column.GetValue(i).ToString(); - - _chunk.Next(); - Assert.AreEqual(valueFromRecordBatch, _chunk.ExtractCell(0).SafeToString()); + TestExtractCell(testValues, sfType, scale, (long)Math.Pow(10, scale)); } } + + [Test] + public void TestExtractCellReturnsNumber32() + { + var testValues = new int[] { 0, 100, -100, Int32.MaxValue, Int32.MinValue }; + var sfType = SFDataType.FIXED; + for (var scale = 0; scale <= 9; ++scale) + { + TestExtractCell(testValues, sfType, scale, (long)Math.Pow(10, scale)); + } + } + [Test] - public void TestExtractCellThrowsOutOfRangeException() + public void TestExtractCellReturnsNumber16() { - _chunk = new ArrowResultChunk(_recordBatchOne); + var testValues = new short[] { 0, 100, -100, Int16.MaxValue, Int16.MinValue }; + var sfType = SFDataType.FIXED; - // move to the end of the batch - while (_chunk.Next()) {} + for (var scale = 0; scale <= 9; ++scale) + { + TestExtractCell(testValues, sfType, scale, (long)Math.Pow(10, scale)); + } + } + + [Test] + public void TestExtractCellReturnsNumber8() + { + var testValues = new sbyte[] { 0, 127, -128 }; + var sfType = SFDataType.FIXED; - Assert.Throws(() => _chunk.ExtractCell(0).SafeToString()); + for (var scale = 0; scale <= 9; ++scale) + { + TestExtractCell(testValues, sfType, scale, (long)Math.Pow(10, scale)); + } } [Test] - public void TestRowCountReturnsNumberOfRows() + public void TestExtractCellReturnsBoolean() { - _chunk = new ArrowResultChunk(_recordBatchOne); + var testValues = new bool[] { true, false }; + var sfType = SFDataType.BOOLEAN; + var scale = 0; - Assert.AreEqual(RowCountBatchOne, _chunk.RowCount); + TestExtractCell(testValues, sfType, scale); } + + [Test] + public void TestExtractCellReturnsReal() + { + var testValues = new double[] { 0, Double.MinValue, Double.MaxValue }; + var sfType = SFDataType.REAL; + var scale = 0; + TestExtractCell(testValues, sfType, scale); + } + [Test] - public void TestGetChunkIndexReturnsFirstChunk() + public void TestExtractCellReturnsText() { - _chunk = new ArrowResultChunk(_recordBatchOne); + var testValues = new string[] + { + "", + TestDataGenarator.StringWithUnicode + }; + var sfType = SFDataType.TEXT; + var scale = 0; - Assert.AreEqual(0, _chunk.ChunkIndex); + TestExtractCell(testValues, sfType, scale); } + [Test] + public void TestExtractCellReturnsArray() + { + var testValues = new string[] + { + "", + TestDataGenarator.StringWithUnicode + }; + var sfType = SFDataType.ARRAY; + var scale = 0; + + TestExtractCell(testValues, sfType, scale); + } + + [Test] + public void TestExtractCellReturnsBinary() + { + var testValues = new byte[][] + { + new byte[] { }, + new byte[] { 0, 19, 33, 200, 10, 13, 255 } + }; + var sfType = SFDataType.BINARY; + var scale = 0; + + TestExtractCell(testValues, sfType, scale); + } + + [Test] + public void TestExtractCellReturnsDate() + { + var testValues = new DateTime[] + { + DateTime.Parse("2019-01-01"), + DateTime.Parse("0001-01-01"), + DateTime.Parse("9999-12-31") + }; + var sfType = SFDataType.DATE; + var scale = 0; + + TestExtractCell(testValues, sfType, scale); + } + + [Test] + public void TestExtractCellReturnsTime() + { + var testValues = new DateTime[] + { + DateTime.Parse("2019-01-01 12:12:12.1234567"), + DateTime.Parse("0001-01-01 00:00:00.0000000"), + DateTime.Parse("9999-12-31 23:59:59.9999999") + }; + var sfType = SFDataType.TIME; + + for (var scale = 0; scale <= 7; ++scale) + { + var values = TruncateValues(testValues, scale); + TestExtractCell(values, sfType, scale); + } + } + + [Test] + public void TestExtractCellReturnsTimestampTz() + { + var testValues = new DateTimeOffset[] + { + DateTimeOffset.Parse("2019-01-01 12:12:12.1234567 +0500"), + DateTimeOffset.Parse("2019-01-01 12:12:12.1234567 -0500"), + DateTimeOffset.Parse("2019-01-01 12:12:12.1234567 +1400"), + DateTimeOffset.Parse("2019-01-01 12:12:12.1234567 -1400"), + DateTimeOffset.Parse("0001-01-01 00:00:00.0000000 +0000"), + DateTimeOffset.Parse("9999-12-31 23:59:59.9999999 +0000"), + }; + var sfType = SFDataType.TIMESTAMP_TZ; + + for (var scale = 0; scale <= 9; ++scale) + { + var values = TruncateValues(testValues, scale); + TestExtractCell(values, sfType, scale); + } + } + + [Test] + public void TestExtractCellReturnsTimestampLtz() + { + var testValues = new DateTimeOffset[] + { + DateTimeOffset.Parse("2019-01-01 12:12:12.1234567").ToLocalTime(), + DateTimeOffset.Parse("0001-01-01 00:00:00.0000000 +0000").ToLocalTime(), + DateTimeOffset.Parse("9999-12-31 23:59:59.9999999 +0000").ToLocalTime(), + }; + var sfType = SFDataType.TIMESTAMP_LTZ; + + for (var scale = 0; scale <= 9; ++scale) + { + var values = TruncateValues(testValues, scale); + TestExtractCell(values, sfType, scale); + } + } + + [Test] + public void TestExtractCellReturnsTimestampNtz() + { + var testValues = new DateTime[] + { + DateTime.Parse("2019-01-01 12:12:12.1234567"), + DateTime.Parse("0001-01-01 00:00:00.0000000"), + DateTime.Parse("9999-12-31 23:59:59.9999999") + }; + var sfType = SFDataType.TIMESTAMP_NTZ; + + for (var scale = 0; scale <= 9; ++scale) + { + var values = TruncateValues(testValues, scale); + TestExtractCell(values, sfType, scale); + } + } + + void TestExtractCell(IEnumerable testValues, SFDataType sfType, long scale, long divider = 0) + { + var recordBatch = PrepareRecordBatch(sfType, scale, testValues); + var chunk = new ArrowResultChunk(recordBatch); + + foreach (var testValue in testValues) + { + chunk.Next(); + + var expectedValue = (divider == 0) ? testValue : Convert.ToDecimal(testValue) / divider; + Assert.AreEqual(expectedValue, chunk.ExtractCell(0, sfType, scale)); + } + } + public static RecordBatch PrepareRecordBatch(SFDataType sfType, long scale, object values) + { + IArrowArray column = null; + switch (sfType) + { + case SFDataType.FIXED: + switch (values) + { + case decimal[] val: + column = new Decimal128Array.Builder(new Decimal128Type(100, (int)scale)) + .AppendRange(val.Select(v => v / (decimal)Math.Pow(10, scale))) + .Build(); + break; + case long[] val: + column = new Int64Array.Builder().AppendRange(val).Build(); + break; + case int[] val: + column = new Int32Array.Builder().AppendRange(val).Build(); + break; + case short[] val: + column = new Int16Array.Builder().AppendRange(val).Build(); + break; + case sbyte[] val: + column = new Int8Array.Builder().AppendRange(val).Build(); + break; + } + + break; + + case SFDataType.BOOLEAN: + column = new BooleanArray.Builder() + .AppendRange(values as bool[]) + .Build(); + break; + + case SFDataType.REAL: + column = new DoubleArray.Builder() + .AppendRange(values as double[]) + .Build(); + break; + + case SFDataType.TEXT: + case SFDataType.ARRAY: + case SFDataType.VARIANT: + case SFDataType.OBJECT: + switch (values) + { + case string[] arr: + column = new StringArray.Builder() + .AppendRange(arr) + .Build(); + break; + case char[] arr: + column = new StringArray.Builder() + .AppendRange(arr.Select(ch => ch.ToString())) + .Build(); + break; + } + break; + + case SFDataType.BINARY: + column = new BinaryArray.Builder() + .AppendRange(values as byte[][]) + .Build(); + break; + + case SFDataType.DATE: + column = new Date32Array.Builder() + .AppendRange(values as DateTime[]) + .Build(); + break; + + case SFDataType.TIME: + { + var arr = values as DateTime[]; + column = new Int64Array.Builder() + .AppendRange(arr.Select(dt => ConvertTicksToInt64(dt.Ticks, scale))) + .Build(); + break; + } + + case SFDataType.TIMESTAMP_TZ: + { + var arr = values as DateTimeOffset[]; + if (scale <= 3) + { + var structField = new StructType(new[] + { + new Field("value", new Int64Type(), nullable: false), + new Field("timezone", new Int32Type(), nullable: false) + }); + + column = new StructArray(structField, arr.Length, new IArrowArray[] + { + new Int64Array.Builder() + .AppendRange(arr.Select(dt => ConvertTicksToInt64(dt.UtcTicks, scale))) + .Build(), + new Int32Array.Builder() + .AppendRange(arr.Select(dt => (int)(1440 + dt.Offset.TotalMinutes))) + .Build() + }, ArrowBuffer.Empty, nullCount: 0); + } + else + { + var structField = new StructType(new[] + { + new Field("epoch", new Int64Type(), nullable: false), + new Field("fraction", new Int32Type(), nullable: false), + new Field("timezone", new Int32Type(), nullable: false) + }); + + column = new StructArray(structField, arr.Length, new IArrowArray[] + { + new Int64Array.Builder() + .AppendRange(arr.Select(dt => dt.ToUnixTimeSeconds())) + .Build(), + new Int32Array.Builder() + .AppendRange(arr.Select(dt => (int)(100 * (dt.UtcTicks % 10000000)))) + .Build(), + new Int32Array.Builder() + .AppendRange(arr.Select(dt => (int)(1440 + dt.Offset.TotalMinutes))) + .Build() + }, ArrowBuffer.Empty, nullCount: 0); + } + + break; + } + case SFDataType.TIMESTAMP_LTZ: + { + var arr = values as DateTimeOffset[]; + if (scale <= 3) + { + column = new Int64Array.Builder() + .AppendRange(arr.Select(dt => ConvertTicksToInt64(dt.UtcTicks, scale))) + .Build(); + } + else + { + var structField = new StructType(new[] + { + new Field("epoch", new Int64Type(), nullable: false), + new Field("fraction", new Int32Type(), nullable: false) + }); + + column = new StructArray(structField, arr.Length, new IArrowArray[] + { + new Int64Array.Builder() + .AppendRange(arr.Select(dt => dt.ToUnixTimeSeconds())) + .Build(), + new Int32Array.Builder() + .AppendRange(arr.Select(dt => (int)(100 * (dt.UtcTicks % 10000000)))) + .Build() + }, ArrowBuffer.Empty, nullCount: 0); + } + break; + } + case SFDataType.TIMESTAMP_NTZ: + { + var arr = values as DateTime[]; + if (scale <= 3) + { + column = new Int64Array.Builder() + .AppendRange(arr.Select(dt => ConvertTicksToInt64(dt.Ticks, scale))) + .Build(); + } + else + { + var structField = new StructType(new[] + { + new Field("epoch", new Int64Type(), nullable: false), + new Field("fraction", new Int32Type(), nullable: false) + }); + + column = new StructArray(structField, arr.Length, new IArrowArray[] + { + new Int64Array.Builder() + .AppendRange(arr.Select(dt => (dt.Ticks - SFDataConverter.UnixEpoch.Ticks) / (long)10000000)) + .Build(), + new Int32Array.Builder() + .AppendRange(arr.Select(dt => (int)(100 * (dt.Ticks % 10000000)))) + .Build() + }, ArrowBuffer.Empty, nullCount: 0); + } + + break; + } + default: + throw new NotSupportedException(); + } + + return new RecordBatch.Builder() + .Append("TestColumn", false, column) + .Build(); + } + + private static long ConvertTicksToInt64(long ticks, long scale) + { + long ticksFromEpoch = ticks - SFDataConverter.UnixEpoch.Ticks; + if (scale <= 7) + return ticksFromEpoch / (long)Math.Pow(10, 7 - scale); + else + return ticksFromEpoch * (long)Math.Pow(10, scale - 7); + } + + public static DateTime[] TruncateValues(DateTime[] testValues, int scale) + { + DateTime[] ret = new DateTime[testValues.Length]; + for (var i = 0; i < testValues.Length; ++i) + if (scale < 7) + ret[i] = testValues[i].AddTicks(-(testValues[i].Ticks % (long)Math.Pow(10, 7 - scale))); + return ret; + } + + public static DateTimeOffset[] TruncateValues(DateTimeOffset[] testValues, int scale) + { + DateTimeOffset[] ret = new DateTimeOffset[testValues.Length]; + for (var i = 0; i < testValues.Length; ++i) + if (scale < 7) + ret[i] = testValues[i].AddTicks(-(testValues[i].Ticks % (long)Math.Pow(10, 7 - scale))); + return ret; + } } } diff --git a/Snowflake.Data.Tests/UnitTests/ArrowResultSetTest.cs b/Snowflake.Data.Tests/UnitTests/ArrowResultSetTest.cs index b2551d5e1..1bd0974f0 100755 --- a/Snowflake.Data.Tests/UnitTests/ArrowResultSetTest.cs +++ b/Snowflake.Data.Tests/UnitTests/ArrowResultSetTest.cs @@ -3,6 +3,7 @@ */ using System; +using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; @@ -12,6 +13,7 @@ using Apache.Arrow.Ipc; using NUnit.Framework; using Snowflake.Data.Core; +using Snowflake.Data.Tests.Util; namespace Snowflake.Data.Tests.UnitTests { @@ -19,19 +21,16 @@ namespace Snowflake.Data.Tests.UnitTests class ArrowResultSetTest { private const int RowCount = 10; + private const int ColumnIndex = 0; + private RecordBatch _recordBatch; private ArrowResultSet _arrowResultSet; [SetUp] public void BeforeTest() { - _recordBatch = new RecordBatch.Builder() - .Append("Col_Int32", false, col => col.Int32(array => array.AppendRange(Enumerable.Range(1, RowCount)))) - .Build(); - var responseData = PrepareResponseData(_recordBatch); - var sfStatement = PrepareStatement(); - - _arrowResultSet = new ArrowResultSet(responseData, sfStatement, new CancellationToken()); + // by default generate Int32 values from 1 to RowCount + PrepareTestCase(SFDataType.FIXED, 0, Enumerable.Range(1, RowCount).ToArray()); } [Test] @@ -75,13 +74,8 @@ public void TestHasRowsReturnsTrueIfRowExists() [Test] public void TestHasRowsReturnsFalseIfNoRows() { - _recordBatch = new RecordBatch.Builder() - .Append("Col_Int32", false, col => col.Int32(array => array.Clear())) - .Build(); - var responseData = PrepareResponseData(_recordBatch); - var sfStatement = PrepareStatement(); + PrepareTestCase(SFDataType.FIXED, 0, new sbyte[]{}); - _arrowResultSet = new ArrowResultSet(responseData, sfStatement, new CancellationToken()); Assert.IsFalse(_arrowResultSet.HasRows()); } @@ -119,18 +113,322 @@ public void TestRewindReturnsTrueForThirdRowAndMovesToFirstRow() } [Test] - public void TestGetObjectInternalReturnsProperValuesForFirstColumn() + public void TestGetDecimal() { - const int ColumnIndex = 0; - var columnValues = (Int32Array)_recordBatch.Column(ColumnIndex); - for (var i = 0; i < RowCount; ++i) + var testValues = new decimal[] { 0, 100, -100, Decimal.MaxValue, Decimal.MinValue }; + + TestGetNumber(testValues); + } + + [Test] + public void TestGetNumber64() + { + var testValues = new long[] { 0, 100, -100, Int64.MaxValue, Int64.MinValue }; + + TestGetNumber(testValues); + } + + [Test] + public void TestGetNumber32() + { + var testValues = new int[] { 0, 100, -100, Int32.MaxValue, Int32.MinValue }; + + TestGetNumber(testValues); + } + + [Test] + public void TestGetNumber16() + { + var testValues = new short[] { 0, 100, -100, Int16.MaxValue, Int16.MinValue }; + + TestGetNumber(testValues); + } + + [Test] + public void TestGetNumber8() + { + var testValues = new sbyte[] { 0, 127, -128 }; + + TestGetNumber(testValues); + } + + private void TestGetNumber(IEnumerable testValues) + { + for (var scale = 0; scale <= 9; ++scale) + { + PrepareTestCase(SFDataType.FIXED, scale, testValues); + + foreach (var testValue in testValues) + { + _arrowResultSet.Next(); + + var expectedValue = Convert.ToDecimal(testValue) / (decimal)Math.Pow(10, scale); + Assert.AreEqual(expectedValue, _arrowResultSet.GetValue(ColumnIndex)); + Assert.AreEqual(expectedValue, _arrowResultSet.GetDecimal(ColumnIndex)); + Assert.AreEqual(expectedValue, _arrowResultSet.GetDouble(ColumnIndex)); + Assert.AreEqual(expectedValue, _arrowResultSet.GetFloat(ColumnIndex)); + + if (expectedValue >= Int64.MinValue && expectedValue <= Int64.MaxValue) + { + // get integer value + long expectedInteger = (long)expectedValue; + + Assert.AreEqual(expectedInteger, _arrowResultSet.GetInt64(ColumnIndex)); + if (expectedInteger >= Int32.MinValue && expectedInteger <= Int32.MaxValue) + Assert.AreEqual(expectedInteger, _arrowResultSet.GetInt32(ColumnIndex)); + else + Assert.Throws(() => _arrowResultSet.GetInt32(ColumnIndex)); + if (expectedInteger >= Int16.MinValue && expectedInteger <= Int16.MaxValue) + Assert.AreEqual(expectedInteger, _arrowResultSet.GetInt16(ColumnIndex)); + else + Assert.Throws(() => _arrowResultSet.GetInt16(ColumnIndex)); + if (expectedInteger >= 0 && expectedInteger <= 255) + Assert.AreEqual(expectedInteger, _arrowResultSet.GetByte(ColumnIndex)); + else + Assert.Throws(() => _arrowResultSet.GetByte(ColumnIndex)); + } + } + } + } + + [Test] + public void TestGetBoolean() + { + var testValues = new bool[] { true, false }; + + PrepareTestCase(SFDataType.BOOLEAN, 0, testValues); + + foreach (var testValue in testValues) + { + _arrowResultSet.Next(); + Assert.AreEqual(testValue, _arrowResultSet.GetValue(ColumnIndex)); + Assert.AreEqual(testValue, _arrowResultSet.GetBoolean(ColumnIndex)); + } + } + + [Test] + public void TestGetReal() + { + var testValues = new double[] { 0, Double.MinValue, Double.MaxValue }; + + PrepareTestCase(SFDataType.REAL, 0, testValues); + + foreach (var testValue in testValues) + { + _arrowResultSet.Next(); + Assert.AreEqual(testValue, _arrowResultSet.GetValue(ColumnIndex)); + Assert.AreEqual(testValue, _arrowResultSet.GetDouble(ColumnIndex)); + } + } + + [Test] + public void TestGetText() + { + var testValues = new string[] + { + "", + TestDataGenarator.StringWithUnicode + }; + + PrepareTestCase(SFDataType.TEXT, 0, testValues); + + foreach (var testValue in testValues) + { + _arrowResultSet.Next(); + Assert.AreEqual(testValue, _arrowResultSet.GetValue(ColumnIndex)); + Assert.AreEqual(testValue, _arrowResultSet.GetString(ColumnIndex)); + } + } + + [Test] + public void TestGetTextWithOneChar() + { + var testValues = + TestDataGenarator.AsciiCodes.ToCharArray() + .Append(TestDataGenarator.SnowflakeUnicode) + .ToArray(); + + PrepareTestCase(SFDataType.TEXT, 0, testValues); + + foreach (var testValue in testValues) + { + _arrowResultSet.Next(); + Assert.AreEqual(testValue, _arrowResultSet.GetChar(ColumnIndex)); + } + } + + [Test] + public void TestGetArray() + { + var testValues = new string[] + { + "", + TestDataGenarator.StringWithUnicode + }; + + PrepareTestCase(SFDataType.ARRAY, 0, testValues); + + foreach (var testValue in testValues) + { + _arrowResultSet.Next(); + Assert.AreEqual(testValue, _arrowResultSet.GetValue(ColumnIndex)); + char[] buffer = new char[1000]; + var len = _arrowResultSet.GetChars(ColumnIndex, 0, buffer, 0, buffer.Length); + var str = new String(buffer, 0, (int)len); + Assert.AreEqual(testValue, str); + Assert.AreEqual(testValue.Length, str.Length); + } + } + + [Test] + public void TestGetBinary() + { + var testValues = new byte[][] + { + new byte[] { }, + new byte[] { 0, 19, 33, 200, 10, 13, 255 } + }; + + PrepareTestCase(SFDataType.BINARY, 0, testValues); + foreach (var testValue in testValues) + { + _arrowResultSet.Next(); + Assert.AreEqual(testValue, _arrowResultSet.GetValue(ColumnIndex)); + byte[] buffer = new byte[100]; + var len = _arrowResultSet.GetBytes(ColumnIndex, 0, buffer, 0, buffer.Length); + Assert.AreEqual(testValue.Length, len); + for (var j = 0; j < len; ++j) + Assert.AreEqual(testValue[j], buffer[j], "position " + j); + } + } + + [Test] + public void TestGetDate() + { + var testValues = new DateTime[] + { + DateTime.Parse("2019-01-01"), + DateTime.Parse("0001-01-01"), + DateTime.Parse("9999-12-31") + }; + + PrepareTestCase(SFDataType.DATE, 0, testValues); + + foreach (var testValue in testValues) { _arrowResultSet.Next(); - Assert.AreEqual(columnValues.GetValue(i).ToString(), _arrowResultSet.getObjectInternal(ColumnIndex).ToString()); + Assert.AreEqual(testValue, _arrowResultSet.GetValue(ColumnIndex)); + Assert.AreEqual(testValue, _arrowResultSet.GetDateTime(ColumnIndex)); } } - private QueryExecResponseData PrepareResponseData(RecordBatch recordBatch) + [Test] + public void TestGetTime() + { + var testValues = new DateTime[] + { + DateTime.Parse("2019-01-01 12:12:12.1234567"), + DateTime.Parse("0001-01-01 00:00:00.0000000"), + DateTime.Parse("9999-12-31 23:59:59.9999999") + }; + + for (var scale = 0; scale <= 7; ++scale) + { + var values = ArrowResultChunkTest.TruncateValues(testValues, scale); + PrepareTestCase(SFDataType.TIME, scale, values); + + foreach (var testValue in values) + { + _arrowResultSet.Next(); + Assert.AreEqual(testValue, _arrowResultSet.GetValue(ColumnIndex)); + Assert.AreEqual(testValue, _arrowResultSet.GetDateTime(ColumnIndex)); + } + } + } + + [Test] + public void TestGetTimestampTz() + { + var testValues = new DateTimeOffset[] + { + DateTimeOffset.Parse("2019-01-01 12:12:12.1234567 +0500"), + DateTimeOffset.Parse("2019-01-01 12:12:12.1234567 -0500"), + DateTimeOffset.Parse("2019-01-01 12:12:12.1234567 +1400"), + DateTimeOffset.Parse("2019-01-01 12:12:12.1234567 -1400"), + DateTimeOffset.Parse("0001-01-01 00:00:00.0000000 +0000"), + DateTimeOffset.Parse("9999-12-31 23:59:59.9999999 +0000"), + }; + + for (var scale = 0; scale <= 9; ++scale) + { + var values = ArrowResultChunkTest.TruncateValues(testValues, scale); + PrepareTestCase(SFDataType.TIMESTAMP_TZ, scale, values); + + foreach (var testValue in values) + { + _arrowResultSet.Next(); + Assert.AreEqual(testValue, _arrowResultSet.GetValue(ColumnIndex)); + } + } + } + + [Test] + public void TestGetTimestampLtz() + { + var testValues = new DateTimeOffset[] + { + DateTimeOffset.Parse("2019-01-01 12:12:12.1234567").ToLocalTime(), + DateTimeOffset.Parse("0001-01-01 00:00:00.0000000 +0000").ToLocalTime(), + DateTimeOffset.Parse("9999-12-31 23:59:59.9999999 +0000").ToLocalTime(), + }; + + for (var scale = 0; scale <= 9; ++scale) + { + var values = ArrowResultChunkTest.TruncateValues(testValues, scale); + PrepareTestCase(SFDataType.TIMESTAMP_LTZ, scale, values); + + foreach (var testValue in values) + { + _arrowResultSet.Next(); + Assert.AreEqual(testValue, _arrowResultSet.GetValue(ColumnIndex)); + } + } + } + + [Test] + public void TestGetTimestampNtz() + { + var testValues = new DateTime[] + { + DateTime.Parse("2019-01-01 12:12:12.1234567"), + DateTime.Parse("0001-01-01 00:00:00.0000000"), + DateTime.Parse("9999-12-31 23:59:59.9999999") + }; + + for (var scale = 0; scale <= 9; ++scale) + { + var values = ArrowResultChunkTest.TruncateValues(testValues, scale); + PrepareTestCase(SFDataType.TIMESTAMP_NTZ, scale, values); + + foreach (var testValue in values) + { + _arrowResultSet.Next(); + Assert.AreEqual(testValue, _arrowResultSet.GetValue(ColumnIndex)); + Assert.AreEqual(testValue, _arrowResultSet.GetDateTime(ColumnIndex)); + } + } + } + + private void PrepareTestCase(SFDataType sfType, long scale, object values) + { + _recordBatch = ArrowResultChunkTest.PrepareRecordBatch(sfType, scale, values); + var responseData = PrepareResponseData(_recordBatch, sfType, scale); + var sfStatement = PrepareStatement(); + + _arrowResultSet = new ArrowResultSet(responseData, sfStatement, new CancellationToken()); + } + + private QueryExecResponseData PrepareResponseData(RecordBatch recordBatch, SFDataType sfType, long scale) { return new QueryExecResponseData { @@ -139,7 +437,8 @@ private QueryExecResponseData PrepareResponseData(RecordBatch recordBatch) new ExecResponseRowType { name = col.Name, - type = "TEXT" + type = sfType.ToString(), + scale = scale }).ToList(), parameters = new List(), chunks = null, @@ -166,5 +465,6 @@ private SFStatement PrepareStatement() SFSession session = new SFSession("user=user;password=password;account=account;", null); return new SFStatement(session); } + } } diff --git a/Snowflake.Data.Tests/UnitTests/ChunkDeserializerTest.cs b/Snowflake.Data.Tests/UnitTests/ChunkDeserializerTest.cs index ac0b3ed26..23e47f41e 100644 --- a/Snowflake.Data.Tests/UnitTests/ChunkDeserializerTest.cs +++ b/Snowflake.Data.Tests/UnitTests/ChunkDeserializerTest.cs @@ -16,18 +16,19 @@ namespace Snowflake.Data.Tests.UnitTests [TestFixture, NonParallelizable] class ChunkDeserializerTest { - int ChunkParserVersionDefault = SFConfiguration.Instance().GetChunkParserVersion(); + private int _chunkParserVersionDefault; [SetUp] public void BeforeTest() { + _chunkParserVersionDefault = SFConfiguration.Instance().ChunkParserVersion; SFConfiguration.Instance().ChunkParserVersion = 2; // ChunkDeserializer } [TearDown] public void AfterTest() { - SFConfiguration.Instance().ChunkParserVersion = ChunkParserVersionDefault; // Return to default version + SFConfiguration.Instance().ChunkParserVersion = _chunkParserVersionDefault; // Return to default version } public IChunkParser getParser(string data) diff --git a/Snowflake.Data.Tests/UnitTests/ChunkDownloaderFactoryTest.cs b/Snowflake.Data.Tests/UnitTests/ChunkDownloaderFactoryTest.cs index e2998197a..cf8b952a0 100644 --- a/Snowflake.Data.Tests/UnitTests/ChunkDownloaderFactoryTest.cs +++ b/Snowflake.Data.Tests/UnitTests/ChunkDownloaderFactoryTest.cs @@ -14,14 +14,21 @@ namespace Snowflake.Data.Tests.UnitTests [TestFixture, NonParallelizable] class ChunkDownloaderFactoryTest { - bool UseV2ChunkDownloaderDefault = SFConfiguration.Instance().UseV2ChunkDownloader; - int ChunkDownloaderVersionDefault = SFConfiguration.Instance().GetChunkDownloaderVersion(); + bool _useV2ChunkDownloaderDefault; + int _chunkDownloaderVersionDefault; + + [SetUp] + public void BeforeTest() + { + _useV2ChunkDownloaderDefault = SFConfiguration.Instance().UseV2ChunkDownloader; + _chunkDownloaderVersionDefault = SFConfiguration.Instance().ChunkDownloaderVersion; + } [TearDown] public void AfterTest() { - SFConfiguration.Instance().UseV2ChunkDownloader = UseV2ChunkDownloaderDefault; // Return to default version - SFConfiguration.Instance().ChunkDownloaderVersion = ChunkDownloaderVersionDefault; // Return to default version + SFConfiguration.Instance().UseV2ChunkDownloader = _useV2ChunkDownloaderDefault; // Return to default version + SFConfiguration.Instance().ChunkDownloaderVersion = _chunkDownloaderVersionDefault; // Return to default version } private QueryExecResponseData mockQueryRequestData() diff --git a/Snowflake.Data.Tests/UnitTests/ChunkParserFactoryTest.cs b/Snowflake.Data.Tests/UnitTests/ChunkParserFactoryTest.cs index 0ab79bcfc..80be130dc 100644 --- a/Snowflake.Data.Tests/UnitTests/ChunkParserFactoryTest.cs +++ b/Snowflake.Data.Tests/UnitTests/ChunkParserFactoryTest.cs @@ -17,14 +17,21 @@ namespace Snowflake.Data.Tests.UnitTests [TestFixture, NonParallelizable] class ChunkParserFactoryTest { - bool UseV2JsonParserDefault = SFConfiguration.Instance().UseV2JsonParser; - int ChunkParserVersionDefault = SFConfiguration.Instance().GetChunkParserVersion(); + bool _useV2JsonParserDefault; + int _chunkParserVersionDefault; + + [SetUp] + public void BeforeTest() + { + _useV2JsonParserDefault = SFConfiguration.Instance().UseV2JsonParser; + _chunkParserVersionDefault = SFConfiguration.Instance().ChunkParserVersion; + } [TearDown] public void AfterTest() { - SFConfiguration.Instance().UseV2JsonParser = UseV2JsonParserDefault; // Return to default version - SFConfiguration.Instance().ChunkParserVersion = ChunkParserVersionDefault; // Return to default version + SFConfiguration.Instance().UseV2JsonParser = _useV2JsonParserDefault; // Return to default version + SFConfiguration.Instance().ChunkParserVersion = _chunkParserVersionDefault; // Return to default version } [Test] diff --git a/Snowflake.Data.Tests/UnitTests/ChunkStreamingParserTest.cs b/Snowflake.Data.Tests/UnitTests/ChunkStreamingParserTest.cs index 06e94ed08..e32541440 100644 --- a/Snowflake.Data.Tests/UnitTests/ChunkStreamingParserTest.cs +++ b/Snowflake.Data.Tests/UnitTests/ChunkStreamingParserTest.cs @@ -16,18 +16,19 @@ namespace Snowflake.Data.Tests.UnitTests [TestFixture, NonParallelizable] class ChunkStreamingParserTest { - int ChunkParserVersionDefault = SFConfiguration.Instance().GetChunkParserVersion(); + int _chunkParserVersionDefault; [SetUp] public void BeforeTest() { + _chunkParserVersionDefault = SFConfiguration.Instance().ChunkParserVersion; SFConfiguration.Instance().ChunkParserVersion = 1; // ChunkStreamingParser } [TearDown] public void AfterTest() { - SFConfiguration.Instance().ChunkParserVersion = ChunkParserVersionDefault; // Return to default version + SFConfiguration.Instance().ChunkParserVersion = _chunkParserVersionDefault; // Return to default version } public IChunkParser getParser(string data) diff --git a/Snowflake.Data.Tests/UnitTests/SFFileTransferAgentTests.cs b/Snowflake.Data.Tests/UnitTests/SFFileTransferAgentTests.cs index 81e7b644e..5d81a610e 100644 --- a/Snowflake.Data.Tests/UnitTests/SFFileTransferAgentTests.cs +++ b/Snowflake.Data.Tests/UnitTests/SFFileTransferAgentTests.cs @@ -150,9 +150,9 @@ private void UploadSetUpFile() } - private string GetResultValue(SFBaseResultSet result, SFResultSet.PutGetResponseRowTypeInfo typeInfo) + private string GetResultValue(SFResultSet result, SFResultSet.PutGetResponseRowTypeInfo typeInfo) { - return result.getObjectInternal((int)typeInfo).ToString(); + return result.GetObjectInternal((int)typeInfo).ToString(); } [Test] @@ -170,7 +170,7 @@ public void TestUploadUsingFilepath() // Act _fileTransferAgent.execute(); - SFBaseResultSet result = _fileTransferAgent.result(); + SFResultSet result = _fileTransferAgent.result(); result.Next(); // Assert @@ -198,7 +198,7 @@ public async Task TestUploadAsyncUsingFilepath() // Act await _fileTransferAgent.executeAsync(_cancellationToken).ConfigureAwait(false); - SFBaseResultSet result = _fileTransferAgent.result(); + SFResultSet result = _fileTransferAgent.result(); result.Next(); // Assert @@ -231,7 +231,7 @@ public void TestUploadUsingMemoryStream() // Act _fileTransferAgent.execute(); - SFBaseResultSet result = _fileTransferAgent.result(); + SFResultSet result = _fileTransferAgent.result(); result.Next(); // Assert @@ -264,7 +264,7 @@ public async Task TestUploadAsyncUsingMemoryStream() // Act await _fileTransferAgent.executeAsync(_cancellationToken).ConfigureAwait(false); - SFBaseResultSet result = _fileTransferAgent.result(); + SFResultSet result = _fileTransferAgent.result(); result.Next(); // Assert @@ -294,7 +294,7 @@ public void TestUploadWithGZIPCompression() // Act _fileTransferAgent.execute(); - SFBaseResultSet result = _fileTransferAgent.result(); + SFResultSet result = _fileTransferAgent.result(); result.Next(); // Assert @@ -340,7 +340,7 @@ public void TestUploadWithWilcardInTheFilename() // Act _fileTransferAgent.execute(); - SFBaseResultSet result = _fileTransferAgent.result(); + SFResultSet result = _fileTransferAgent.result(); // Assert for (int index = 0; index < numberOfFiles; index++) @@ -390,7 +390,7 @@ public void TestUploadWithWildcardInTheRootDirectory() // Act _fileTransferAgent.execute(); - SFBaseResultSet result = _fileTransferAgent.result(); + SFResultSet result = _fileTransferAgent.result(); // Assert for (int i = 0; i < numberOfDirectories; i++) @@ -441,7 +441,7 @@ public void TestUploadWithWildcardInTheDirectoryPath() // Act _fileTransferAgent.execute(); - SFBaseResultSet result = _fileTransferAgent.result(); + SFResultSet result = _fileTransferAgent.result(); // Assert for (int i = 0; i < numberOfDirectories; i++) @@ -530,7 +530,7 @@ public void TestDownload() // Act _fileTransferAgent.execute(); - SFBaseResultSet result = _fileTransferAgent.result(); + SFResultSet result = _fileTransferAgent.result(); result.Next(); // Assert @@ -555,7 +555,7 @@ public async Task TestDownloadAsync() // Act await _fileTransferAgent.executeAsync(_cancellationToken).ConfigureAwait(false); - SFBaseResultSet result = _fileTransferAgent.result(); + SFResultSet result = _fileTransferAgent.result(); result.Next(); // Assert diff --git a/Snowflake.Data.Tests/Util/SessionParameterAlterer.cs b/Snowflake.Data.Tests/Util/SessionParameterAlterer.cs new file mode 100644 index 000000000..3a31dd508 --- /dev/null +++ b/Snowflake.Data.Tests/Util/SessionParameterAlterer.cs @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved. + */ + +using System.Data; +using Snowflake.Data.Client; +using Snowflake.Data.Core; + +namespace Snowflake.Data.Tests.Util +{ + class SessionParameterAlterer + { + public static void SetResultFormat(IDbConnection conn, ResultFormat resultFormat) + { + IDbCommand cmd = conn.CreateCommand(); + if (resultFormat == ResultFormat.ARROW) + // ARROW_FORCE to set Arrow format regardless driver version + cmd.CommandText = $"alter session set DOTNET_QUERY_RESULT_FORMAT = ARROW_FORCE"; + else + cmd.CommandText = $"alter session set DOTNET_QUERY_RESULT_FORMAT = {resultFormat}"; + try + { + cmd.ExecuteNonQuery(); + } + catch (SnowflakeDbException ex) + { + if (ex.Message.Contains("invalid parameter")) + return; + throw ex; + } + } + + public static void RestoreResultFormat(IDbConnection conn) + { + IDbCommand cmd = conn.CreateCommand(); + cmd.CommandText = "alter session set DOTNET_QUERY_RESULT_FORMAT = default"; + try + { + cmd.ExecuteNonQuery(); + } + catch (SnowflakeDbException ex) + { + if (ex.Message.Contains("invalid parameter")) + return; + throw ex; + } + } + + public static void SetPrefetchThreads(IDbConnection conn, int prefetchThreads) + { + IDbCommand cmd = conn.CreateCommand(); + cmd.CommandText = $"alter session set CLIENT_PREFETCH_THREADS = {prefetchThreads}"; + cmd.ExecuteNonQuery(); + } + + public static void RestorePrefetchThreads(IDbConnection conn) + { + IDbCommand cmd = conn.CreateCommand(); + cmd.CommandText = "alter session set CLIENT_PREFETCH_THREADS = default"; + cmd.ExecuteNonQuery(); + } + } +} diff --git a/Snowflake.Data.Tests/Util/TestDataGenarator.cs b/Snowflake.Data.Tests/Util/TestDataGenarator.cs index ded300a2b..27dda5ab0 100644 --- a/Snowflake.Data.Tests/Util/TestDataGenarator.cs +++ b/Snowflake.Data.Tests/Util/TestDataGenarator.cs @@ -3,6 +3,7 @@ */ using System; +using System.Linq; namespace Snowflake.Data.Tests.Util { @@ -16,6 +17,11 @@ public class TestDataGenarator private static string s_digitChars = "0" + s_nonZeroDigits; private static string s_letterChars = s_lowercaseChars + s_uppercaseChars; private static string s_alphanumericChars = s_letterChars + s_digitChars; + + public static string AsciiCodes => new String(Enumerable.Range(0, 256).Select(ch => (char)ch).ToArray()); + public static char SnowflakeUnicode => '\u2744'; + public static string EmojiUnicode => "\uD83D\uDE00"; + public static string StringWithUnicode => AsciiCodes + SnowflakeUnicode + EmojiUnicode; public static bool NextBool() { diff --git a/Snowflake.Data/Client/SnowflakeDbConnection.cs b/Snowflake.Data/Client/SnowflakeDbConnection.cs index 5f66d340c..36c35011f 100755 --- a/Snowflake.Data/Client/SnowflakeDbConnection.cs +++ b/Snowflake.Data/Client/SnowflakeDbConnection.cs @@ -176,7 +176,7 @@ public override async Task CloseAsync() } #endif - public Task CloseAsync(CancellationToken cancellationToken) + public virtual Task CloseAsync(CancellationToken cancellationToken) { logger.Debug("Close Connection."); TaskCompletionSource taskCompletionSource = new TaskCompletionSource(); diff --git a/Snowflake.Data/Client/SnowflakeDbDataReader.cs b/Snowflake.Data/Client/SnowflakeDbDataReader.cs index 6c0db7a96..20ca1f7ba 100755 --- a/Snowflake.Data/Client/SnowflakeDbDataReader.cs +++ b/Snowflake.Data/Client/SnowflakeDbDataReader.cs @@ -29,6 +29,8 @@ public class SnowflakeDbDataReader : DbDataReader private int RecordsAffectedInternal; + internal ResultFormat ResultFormat => resultSet.ResultFormat; + internal SnowflakeDbDataReader(SnowflakeDbCommand command, SFBaseResultSet resultSet) { this.dbCommand = command; @@ -137,64 +139,53 @@ private DataTable PopulateSchemaTable(SFBaseResultSet resultSet) public override bool GetBoolean(int ordinal) { - return resultSet.GetValue(ordinal); + return resultSet.GetBoolean(ordinal); } public override byte GetByte(int ordinal) { - byte[] bytes = resultSet.GetValue(ordinal); - return bytes[0]; + return resultSet.GetByte(ordinal); } public override long GetBytes(int ordinal, long dataOffset, byte[] buffer, int bufferOffset, int length) { - return readSubset(ordinal, dataOffset, buffer, bufferOffset, length); + return resultSet.GetBytes(ordinal, dataOffset, buffer, bufferOffset, length); } public override char GetChar(int ordinal) { - string val = resultSet.GetString(ordinal); - return val[0]; + return resultSet.GetChar(ordinal); } public override long GetChars(int ordinal, long dataOffset, char[] buffer, int bufferOffset, int length) { - return readSubset(ordinal, dataOffset, buffer, bufferOffset, length); + return resultSet.GetChars(ordinal, dataOffset, buffer, bufferOffset, length); } public override string GetDataTypeName(int ordinal) { - return resultSet.sfResultSetMetaData.getColumnTypeByIndex(ordinal).ToString(); + resultSet.ThrowIfOutOfBounds(ordinal); + return resultSet.sfResultSetMetaData.GetColumnTypeByIndex(ordinal).ToString(); } public override DateTime GetDateTime(int ordinal) { - return resultSet.GetValue(ordinal); + return resultSet.GetDateTime(ordinal); } - /// - /// Retrieves the value of the specified column as a TimeSpan object. - /// - /// The zero-based column ordinal. - /// The value of the specified column as a TimeSpan. - /// The specified cast is not valid. - /// - /// Call IsDBNull to check for null values before calling this method, because TimeSpan - /// objects are not nullable. - /// public TimeSpan GetTimeSpan(int ordinal) { - return resultSet.GetValue(ordinal); + return resultSet.GetTimeSpan(ordinal); } public override decimal GetDecimal(int ordinal) { - return resultSet.GetValue(ordinal); + return resultSet.GetDecimal(ordinal); } public override double GetDouble(int ordinal) { - return resultSet.GetValue(ordinal); + return resultSet.GetDouble(ordinal); } public override IEnumerator GetEnumerator() @@ -204,42 +195,44 @@ public override IEnumerator GetEnumerator() public override Type GetFieldType(int ordinal) { - return resultSet.sfResultSetMetaData.getCSharpTypeByIndex(ordinal); + resultSet.ThrowIfOutOfBounds(ordinal); + return resultSet.sfResultSetMetaData.GetCSharpTypeByIndex(ordinal); } public override float GetFloat(int ordinal) { - return resultSet.GetValue(ordinal); + return resultSet.GetFloat(ordinal); } public override Guid GetGuid(int ordinal) { - return resultSet.GetValue(ordinal); + return resultSet.GetGuid(ordinal); } public override short GetInt16(int ordinal) { - return resultSet.GetValue(ordinal); + return resultSet.GetInt16(ordinal); } public override int GetInt32(int ordinal) { - return resultSet.GetValue(ordinal); + return resultSet.GetInt32(ordinal); } public override long GetInt64(int ordinal) { - return resultSet.GetValue(ordinal); + return resultSet.GetInt64(ordinal); } public override string GetName(int ordinal) { - return resultSet.sfResultSetMetaData.getColumnNameByIndex(ordinal); + resultSet.ThrowIfOutOfBounds(ordinal); + return resultSet.sfResultSetMetaData.GetColumnNameByIndex(ordinal); } public override int GetOrdinal(string name) { - return resultSet.sfResultSetMetaData.getColumnIndexByName(name); + return resultSet.sfResultSetMetaData.GetColumnIndexByName(name); } public override string GetString(int ordinal) @@ -308,73 +301,5 @@ public override void Close() isClosed = true; } - // - // Summary: - // Reads a subset of data starting at location indicated by dataOffset into the buffer, - // starting at the location indicated by bufferOffset. - // - // Parameters: - // ordinal: - // The zero-based column ordinal. - // - // dataOffset: - // The index within the data from which to begin the read operation. - // - // buffer: - // The buffer into which to copy the data. - // - // bufferOffset: - // The index with the buffer to which the data will be copied. - // - // length: - // The maximum number of elements to read. - // - // Returns: - // The actual number of elements read. - private long readSubset(int ordinal, long dataOffset, T[] buffer, int bufferOffset, int length) - { - if (dataOffset < 0) - { - throw new ArgumentOutOfRangeException("dataOffset", "Non negative number is required."); - } - - if (bufferOffset < 0) - { - throw new ArgumentOutOfRangeException("bufferOffset", "Non negative number is required."); - } - - if ((null != buffer) && (bufferOffset > buffer.Length)) - { - throw new System.ArgumentException("Destination buffer is not long enough. " + - "Check the buffer offset, length, and the buffer's lower bounds.", "buffer"); - } - - T[] data = resultSet.GetValue(ordinal); - - // https://docs.microsoft.com/en-us/dotnet/api/system.data.idatarecord.getbytes?view=net-5.0#remarks - // If you pass a buffer that is null, GetBytes returns the length of the row in bytes. - // https://docs.microsoft.com/en-us/dotnet/api/system.data.idatarecord.getchars?view=net-5.0#remarks - // If you pass a buffer that is null, GetChars returns the length of the field in characters. - if (null == buffer) - { - return data.Length; - } - - if (dataOffset > data.Length) - { - throw new System.ArgumentException("Source data is not long enough. " + - "Check the data offset, length, and the data's lower bounds." ,"dataOffset"); - } - else - { - // How much data is available after the offset - long dataLength = data.Length - dataOffset; - // How much data to read - long elementsRead = Math.Min(length, dataLength); - Array.Copy(data, dataOffset, buffer, bufferOffset, elementsRead); - - return elementsRead; - } - } } } diff --git a/Snowflake.Data/Core/ArrowResultChunk.cs b/Snowflake.Data/Core/ArrowResultChunk.cs index b226ec129..c743aabb2 100755 --- a/Snowflake.Data/Core/ArrowResultChunk.cs +++ b/Snowflake.Data/Core/ArrowResultChunk.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; -using System.Text; using Apache.Arrow; using Apache.Arrow.Types; @@ -12,17 +11,32 @@ namespace Snowflake.Data.Core { internal class ArrowResultChunk : BaseResultChunk { - internal override ResultFormat Format => ResultFormat.ARROW; + internal override ResultFormat ResultFormat => ResultFormat.ARROW; + + private static readonly DateTimeOffset s_epochDate = SFDataConverter.UnixEpoch; + + private static readonly long[] s_powersOf10 = { + 1, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000, + 1000000000 + }; public List RecordBatch { get; set; } private int _currentBatchIndex = 0; private int _currentRecordIndex = -1; - + public ArrowResultChunk(RecordBatch recordBatch) { - RecordBatch = new List{recordBatch}; - + RecordBatch = new List { recordBatch }; + RowCount = recordBatch.Length; ColumnCount = recordBatch.ColumnCount; ChunkIndex = 0; @@ -36,7 +50,7 @@ public ArrowResultChunk(int columnCount) ColumnCount = columnCount; ChunkIndex = 0; } - + public void AddRecordBatch(RecordBatch recordBatch) { RecordBatch.Add(recordBatch); @@ -53,10 +67,13 @@ internal override void Reset(ExecResponseChunk chunkInfo, int chunkIndex) internal override bool Next() { + if (_currentBatchIndex >= RecordBatch.Count) + return false; + _currentRecordIndex += 1; if (_currentRecordIndex < RecordBatch[_currentBatchIndex].Length) return true; - + _currentBatchIndex += 1; _currentRecordIndex = 0; @@ -65,10 +82,13 @@ internal override bool Next() internal override bool Rewind() { + if (_currentRecordIndex == -1) + return false; + _currentRecordIndex -= 1; if (_currentRecordIndex >= 0) return true; - + _currentBatchIndex -= 1; if (_currentBatchIndex >= 0) @@ -79,42 +99,159 @@ internal override bool Rewind() return false; } - + + [Obsolete("ExtractCell with rowIndex is deprecated", false)] public override UTF8Buffer ExtractCell(int rowIndex, int columnIndex) { - _currentBatchIndex = 0; - _currentRecordIndex = rowIndex; - while (_currentRecordIndex >= RecordBatch[_currentBatchIndex].Length) - { - _currentRecordIndex -= RecordBatch[_currentBatchIndex].Length; - _currentBatchIndex += 1; - } - - return ExtractCell(columnIndex); + throw new NotSupportedException(); } public override UTF8Buffer ExtractCell(int columnIndex) + { + throw new NotSupportedException(); + } + + public object ExtractCell(int columnIndex, SFDataType srcType, long scale) { var column = RecordBatch[_currentBatchIndex].Column(columnIndex); - string stringBuffer; - switch (column.Data.DataType.TypeId) + if (column.IsNull(_currentRecordIndex)) + return DBNull.Value; + + switch (srcType) { - case ArrowTypeId.Int32: - stringBuffer = ((Int32Array)column).GetValue(_currentRecordIndex).ToString(); + case SFDataType.FIXED: + // Snowflake data types that are fixed-point numbers will fall into this category + // e.g. NUMBER, DECIMAL/NUMERIC, INT/INTEGER + switch (column.Data.DataType.TypeId) + { + case ArrowTypeId.Int8: + if (scale == 0) + return ((Int8Array)column).GetValue(_currentRecordIndex); + else + return ((Int8Array)column).GetValue(_currentRecordIndex) / (decimal)s_powersOf10[scale]; + case ArrowTypeId.Int16: + if (scale == 0) + return ((Int16Array)column).GetValue(_currentRecordIndex); + else + return ((Int16Array)column).GetValue(_currentRecordIndex) / (decimal)s_powersOf10[scale]; + case ArrowTypeId.Int32: + if (scale == 0) + return ((Int32Array)column).GetValue(_currentRecordIndex); + else + return ((Int32Array)column).GetValue(_currentRecordIndex) / (decimal)s_powersOf10[scale]; + case ArrowTypeId.Int64: + if (scale == 0) + return ((Int64Array)column).GetValue(_currentRecordIndex); + else + return ((Int64Array)column).GetValue(_currentRecordIndex) / (decimal)s_powersOf10[scale]; + case ArrowTypeId.Decimal128: + return ((Decimal128Array)column).GetValue(_currentRecordIndex); + } break; - - // TODO in SNOW-893834 - other types - - default: - throw new NotImplementedException(); + + case SFDataType.BOOLEAN: + return ((BooleanArray)column).GetValue(_currentRecordIndex); + case SFDataType.REAL: + // Snowflake data types that are floating-point numbers will fall in this category + // e.g. FLOAT/REAL/DOUBLE + return ((DoubleArray)column).GetValue(_currentRecordIndex); + case SFDataType.TEXT: + case SFDataType.ARRAY: + case SFDataType.VARIANT: + case SFDataType.OBJECT: + return ((StringArray)column).GetString(_currentRecordIndex); + case SFDataType.BINARY: + return ((BinaryArray)column).GetBytes(_currentRecordIndex).ToArray(); + case SFDataType.DATE: + return ((Date32Array)column).GetDateTime(_currentRecordIndex); + case SFDataType.TIME: + { + var value = column.Data.DataType.TypeId == ArrowTypeId.Int32 + ? ((Int32Array)column).GetValue(_currentRecordIndex) + : ((Int64Array)column).GetValue(_currentRecordIndex); + if (value == null) + return null; + if (scale == 0) + return DateTimeOffset.FromUnixTimeSeconds(value.Value).DateTime; + if (scale <= 3) + return DateTimeOffset.FromUnixTimeMilliseconds(value.Value * s_powersOf10[3 - scale]) + .DateTime; + if (scale <= 7) + return s_epochDate.AddTicks(value.Value * s_powersOf10[7 - scale]).DateTime; + return s_epochDate.AddTicks(value.Value / s_powersOf10[scale - 7]).DateTime; + } + case SFDataType.TIMESTAMP_TZ: + if (((StructArray)column).Fields.Count == 2) + { + var value = ((Int64Array)((StructArray)column).Fields[0]).GetValue(_currentRecordIndex); + var timezone = ((Int32Array)((StructArray)column).Fields[1]).GetValue(_currentRecordIndex); + if (value == null || timezone == null) + return null; + var epoch = ExtractEpoch(value.Value, scale); + var fraction = ExtractFraction(value.Value, scale); + return s_epochDate.AddSeconds(epoch).AddTicks(fraction / 100).ToOffset(TimeSpan.FromMinutes(timezone.Value - 1440)); + } + else + { + var epoch = ((Int64Array)((StructArray)column).Fields[0]).GetValue(_currentRecordIndex); + var fraction = ((Int32Array)((StructArray)column).Fields[1]).GetValue(_currentRecordIndex); + var timezone = ((Int32Array)((StructArray)column).Fields[2]).GetValue(_currentRecordIndex); + if (epoch == null || fraction == null || timezone == null) + return null; + return s_epochDate.AddSeconds(epoch.Value).AddTicks(fraction.Value / 100).ToOffset(TimeSpan.FromMinutes(timezone.Value - 1440)); + } + + case SFDataType.TIMESTAMP_LTZ: + if (column.Data.DataType.TypeId == ArrowTypeId.Struct) + { + var epoch = ((Int64Array)((StructArray)column).Fields[0]).GetValue(_currentRecordIndex); + var fraction = ((Int32Array)((StructArray)column).Fields[1]).GetValue(_currentRecordIndex); + if (epoch == null || fraction == null) + return null; + return s_epochDate.AddSeconds(epoch.Value).AddTicks(fraction.Value / 100).ToLocalTime(); + } + else + { + var value = ((Int64Array)column).GetValue(_currentRecordIndex); + if (value == null) + return null; + var epoch = ExtractEpoch(value.Value, scale); + var fraction = ExtractFraction(value.Value, scale); + return s_epochDate.AddSeconds(epoch).AddTicks(fraction / 100).ToLocalTime(); + } + + case SFDataType.TIMESTAMP_NTZ: + if (column.Data.DataType.TypeId == ArrowTypeId.Struct) + { + var epoch = ((Int64Array)((StructArray)column).Fields[0]).GetValue(_currentRecordIndex); + var fraction = ((Int32Array)((StructArray)column).Fields[1]).GetValue(_currentRecordIndex); + if (epoch == null || fraction == null) + return null; + return s_epochDate.AddSeconds(epoch.Value).AddTicks(fraction.Value / 100).DateTime; + } + else + { + var value = ((Int64Array)column).GetValue(_currentRecordIndex); + if (value == null) + return null; + var epoch = ExtractEpoch(value.Value, scale); + var fraction = ExtractFraction(value.Value, scale); + return s_epochDate.AddSeconds(epoch).AddTicks(fraction / 100).DateTime; + } } - - if (stringBuffer == null) - return null; - - return new UTF8Buffer(Encoding.UTF8.GetBytes(stringBuffer)); + + return null; + } + + private long ExtractEpoch(long value, long scale) + { + return value / s_powersOf10[scale]; } - } + private long ExtractFraction(long value, long scale) + { + return ((value % s_powersOf10[scale]) * s_powersOf10[9 - scale]); + } + } } diff --git a/Snowflake.Data/Core/ArrowResultSet.cs b/Snowflake.Data/Core/ArrowResultSet.cs index 5c3103320..496d2867b 100755 --- a/Snowflake.Data/Core/ArrowResultSet.cs +++ b/Snowflake.Data/Core/ArrowResultSet.cs @@ -4,6 +4,7 @@ using System; using System.IO; +using System.Text; using System.Threading; using System.Threading.Tasks; using Apache.Arrow.Ipc; @@ -14,6 +15,8 @@ namespace Snowflake.Data.Core { class ArrowResultSet : SFBaseResultSet { + internal override ResultFormat ResultFormat => ResultFormat.ARROW; + private static readonly SFLogger s_logger = SFLoggerFactory.GetLogger(); private readonly int _totalChunkCount; @@ -25,14 +28,21 @@ public ArrowResultSet(QueryExecResponseData responseData, SFStatement sfStatemen columnCount = responseData.rowType.Count; try { - using (var stream = new MemoryStream(Convert.FromBase64String(responseData.rowsetBase64))) + if (responseData.rowsetBase64.Length > 0) { - using (var reader = new ArrowStreamReader(stream)) + using (var stream = new MemoryStream(Convert.FromBase64String(responseData.rowsetBase64))) { - var recordBatch = reader.ReadNextRecordBatch(); - _currentChunk = new ArrowResultChunk(recordBatch); + using (var reader = new ArrowStreamReader(stream)) + { + var recordBatch = reader.ReadNextRecordBatch(); + _currentChunk = new ArrowResultChunk(recordBatch); + } } } + else + { + _currentChunk = new ArrowResultChunk(columnCount); + } this.sfStatement = sfStatement; UpdateSessionStatus(responseData); @@ -131,23 +141,280 @@ internal override bool Rewind() return false; } - internal override UTF8Buffer getObjectInternal(int columnIndex) + private object GetObjectInternal(int ordinal) { ThrowIfClosed(); + ThrowIfOutOfBounds(ordinal); + + var type = sfResultSetMetaData.GetTypesByIndex(ordinal).Item1; + var scale = sfResultSetMetaData.GetScaleByIndex(ordinal); + + var value = ((ArrowResultChunk)_currentChunk).ExtractCell(ordinal, type, (int)scale); + + return value ?? DBNull.Value; + + } + + internal override object GetValue(int ordinal) + { + var value = GetObjectInternal(ordinal); + if (value == DBNull.Value) + return value; + + if (value is decimal ret) + return ret; + + var dstType = sfResultSetMetaData.GetCSharpTypeByIndex(ordinal); + + return Convert.ChangeType(value, dstType); + } + + internal override bool IsDBNull(int ordinal) + { + return GetObjectInternal(ordinal) == DBNull.Value; + } + + internal override bool GetBoolean(int ordinal) + { + return (bool)GetObjectInternal(ordinal); + } + + internal override byte GetByte(int ordinal) + { + var value = GetObjectInternal(ordinal); + checked + { + switch (value) + { + case decimal ret: return (byte)ret; + case long ret: return (byte)ret; + case int ret: return (byte)ret; + case short ret: return (byte)ret; + case sbyte ret: return (byte)ret; + default: return (byte)value; + } + } + } + + internal override long GetBytes(int ordinal, long dataOffset, byte[] buffer, int bufferOffset, int length) + { + return ReadSubset(ordinal, dataOffset, buffer, bufferOffset, length); + } + + internal override char GetChar(int ordinal) + { + return ((string)GetObjectInternal(ordinal))[0]; + } + + internal override long GetChars(int ordinal, long dataOffset, char[] buffer, int bufferOffset, int length) + { + return ReadSubset(ordinal, dataOffset, buffer, bufferOffset, length); + } + + internal override DateTime GetDateTime(int ordinal) + { + var value = GetObjectInternal(ordinal); + if (value == DBNull.Value) + return (DateTime)value; - if (columnIndex < 0 || columnIndex >= columnCount) + switch (value) { - throw new SnowflakeDbException(SFError.COLUMN_INDEX_OUT_OF_BOUND, columnIndex); + case DateTime ret: + return ret; + case DateTimeOffset ret: + return ret.DateTime; } + return (DateTime)Convert.ChangeType(value, typeof(DateTime)); + } + + internal override TimeSpan GetTimeSpan(int ordinal) + { + var value = GetObjectInternal(ordinal); + if (value == DBNull.Value) + return (TimeSpan)value; + var type = sfResultSetMetaData.GetColumnTypeByIndex(ordinal); + if (type == SFDataType.TIME && value is DateTime ret) + return TimeSpan.FromTicks(ret.Ticks - SFDataConverter.UnixEpoch.Ticks); + throw new SnowflakeDbException(SFError.INVALID_DATA_CONVERSION, value, type, typeof(TimeSpan)); + } - return _currentChunk.ExtractCell(columnIndex); + internal override decimal GetDecimal(int ordinal) + { + var value = GetObjectInternal(ordinal); + switch (value) + { + case double ret: return (decimal)ret; + case float ret: return (decimal)ret; + case long ret: return ret; + case int ret: return ret; + case short ret: return ret; + case sbyte ret: return ret; + default: return (decimal)value; + } + } + + internal override double GetDouble(int ordinal) + { + var value = GetObjectInternal(ordinal); + switch (value) + { + case float ret: return ret; + case decimal ret: return (double)ret; + case long ret: return ret; + case int ret: return ret; + case short ret: return ret; + case sbyte ret: return ret; + default: return (double)value; + } + } + + internal override float GetFloat(int ordinal) + { + var value = GetObjectInternal(ordinal); + switch (value) + { + case double ret: return (float)ret; + case decimal ret: return (float)ret; + case long ret: return ret; + case int ret: return ret; + case short ret: return ret; + case sbyte ret: return ret; + default: return (float)value; + } } + internal override Guid GetGuid(int ordinal) + { + return new Guid(GetString(ordinal)); + } + + internal override short GetInt16(int ordinal) + { + var value = GetObjectInternal(ordinal); + checked + { + switch (value) + { + case decimal ret: return (short)ret; + case long ret: return (short)ret; + case int ret: return (short)ret; + case sbyte ret: return ret; + default: return (short)value; + } + } + } + + internal override int GetInt32(int ordinal) + { + var value = GetObjectInternal(ordinal); + checked + { + switch (value) + { + case decimal ret: return (int)ret; + case long ret: return (int)ret; + case short ret: return ret; + case sbyte ret: return ret; + default: return (int)value; + } + } + } + + internal override long GetInt64(int ordinal) + { + var value = GetObjectInternal(ordinal); + checked + { + switch (value) + { + case decimal ret: return (long)ret; + case int ret: return ret; + case short ret: return ret; + case sbyte ret: return ret; + default: return (long)value; + } + } + } + + internal override string GetString(int ordinal) + { + var value = GetObjectInternal(ordinal); + if (value == DBNull.Value) + return (string)value; + + var type = sfResultSetMetaData.GetColumnTypeByIndex(ordinal); + switch (value) + { + case string ret: + return ret; + case DateTime ret: + if (type == SFDataType.DATE) + return SFDataConverter.toDateString(ret, sfResultSetMetaData.dateOutputFormat); + break; + } + + return Convert.ToString(value); + } + private void UpdateSessionStatus(QueryExecResponseData responseData) { SFSession session = this.sfStatement.SfSession; session.UpdateDatabaseAndSchema(responseData.finalDatabaseName, responseData.finalSchemaName); session.UpdateSessionParameterMap(responseData.parameters); } + + private long ReadSubset(int ordinal, long dataOffset, T[] buffer, int bufferOffset, int length) where T : struct + { + if (dataOffset < 0) + { + throw new ArgumentOutOfRangeException(nameof(dataOffset), "Non negative number is required."); + } + + if (bufferOffset < 0) + { + throw new ArgumentOutOfRangeException(nameof(bufferOffset), "Non negative number is required."); + } + + if (buffer != null && bufferOffset > buffer.Length) + { + throw new System.ArgumentException( + "Destination buffer is not long enough. Check the buffer offset, length, and the buffer's lower bounds.", + nameof(buffer)); + } + + var value = GetObjectInternal(ordinal); + var type = sfResultSetMetaData.GetColumnTypeByIndex(ordinal); + Array data; + if (type == SFDataType.BINARY) + data = (byte[])value; + else if (typeof(T) == typeof(byte)) + data = Encoding.ASCII.GetBytes(value.ToString()); + else + data = value.ToString().ToCharArray(); + + // https://docs.microsoft.com/en-us/dotnet/api/system.data.idatarecord.getbytes?view=net-5.0#remarks + // If you pass a buffer that is null, GetBytes returns the length of the row in bytes. + // https://docs.microsoft.com/en-us/dotnet/api/system.data.idatarecord.getchars?view=net-5.0#remarks + // If you pass a buffer that is null, GetChars returns the length of the field in characters. + if (buffer == null) + { + return data.Length; + } + + if (dataOffset > data.Length) + { + throw new System.ArgumentException( + "Source data is not long enough. Check the data offset, length, and the data's lower bounds.", + nameof(dataOffset)); + } + + long dataLength = data.Length - dataOffset; + long elementsRead = Math.Min(length, dataLength); + Array.Copy(data, dataOffset, buffer, bufferOffset, elementsRead); + + return elementsRead; + + } + } } diff --git a/Snowflake.Data/Core/BaseResultChunk.cs b/Snowflake.Data/Core/BaseResultChunk.cs index da0591364..e84fa1602 100755 --- a/Snowflake.Data/Core/BaseResultChunk.cs +++ b/Snowflake.Data/Core/BaseResultChunk.cs @@ -2,11 +2,13 @@ * Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. */ +using System; + namespace Snowflake.Data.Core { public abstract class BaseResultChunk : IResultChunk { - internal abstract ResultFormat Format { get; } + internal abstract ResultFormat ResultFormat { get; } public int RowCount { get; protected set; } @@ -22,6 +24,7 @@ public abstract class BaseResultChunk : IResultChunk public int GetChunkIndex() => ChunkIndex; + [Obsolete("ExtractCell with rowIndex is deprecated", false)] public abstract UTF8Buffer ExtractCell(int rowIndex, int columnIndex); public abstract UTF8Buffer ExtractCell(int columnIndex); diff --git a/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs b/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs index 2c00e1604..e5badcf19 100644 --- a/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs +++ b/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs @@ -275,7 +275,7 @@ public async Task executeAsync(CancellationToken cancellationToken) /// Generate the result set based on the file metadata. /// /// The result set containing file status and info - public SFBaseResultSet result() + public SFResultSet result() { // Set the row count using the number of metadata in the result metas TransferMetadata.rowSet = new string[ResultsMetas.Count, 8]; diff --git a/Snowflake.Data/Core/IResultChunk.cs b/Snowflake.Data/Core/IResultChunk.cs index 662bc6999..2ad2bd781 100755 --- a/Snowflake.Data/Core/IResultChunk.cs +++ b/Snowflake.Data/Core/IResultChunk.cs @@ -2,6 +2,8 @@ * Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. */ +using System; + namespace Snowflake.Data.Core { public enum ResultFormat @@ -12,6 +14,7 @@ public enum ResultFormat public interface IResultChunk { + [Obsolete("ExtractCell with rowIndex is deprecated", false)] UTF8Buffer ExtractCell(int rowIndex, int columnIndex); int GetRowCount(); diff --git a/Snowflake.Data/Core/ResultSetUtil.cs b/Snowflake.Data/Core/ResultSetUtil.cs index b16e25372..9d62a17d7 100755 --- a/Snowflake.Data/Core/ResultSetUtil.cs +++ b/Snowflake.Data/Core/ResultSetUtil.cs @@ -28,25 +28,25 @@ internal static int CalculateUpdateCount(this SFBaseResultSet resultSet) resultSet.Next(); for (int i = 0; i < resultSet.columnCount; i++) { - updateCount += resultSet.GetValue(i); + updateCount += resultSet.GetInt64(i); } break; case SFStatementType.COPY: - var index = resultSet.sfResultSetMetaData.getColumnIndexByName("rows_loaded"); + var index = resultSet.sfResultSetMetaData.GetColumnIndexByName("rows_loaded"); if (index >= 0) { resultSet.Next(); - updateCount = resultSet.GetValue(index); + updateCount = resultSet.GetInt64(index); resultSet.Rewind(); } break; case SFStatementType.COPY_UNLOAD: - var rowIndex = resultSet.sfResultSetMetaData.getColumnIndexByName("rows_unloaded"); + var rowIndex = resultSet.sfResultSetMetaData.GetColumnIndexByName("rows_unloaded"); if (rowIndex >= 0) { resultSet.Next(); - updateCount = resultSet.GetValue(rowIndex); + updateCount = resultSet.GetInt64(rowIndex); resultSet.Rewind(); } break; diff --git a/Snowflake.Data/Core/SFBaseResultSet.cs b/Snowflake.Data/Core/SFBaseResultSet.cs index 5b7422c26..348865c1d 100755 --- a/Snowflake.Data/Core/SFBaseResultSet.cs +++ b/Snowflake.Data/Core/SFBaseResultSet.cs @@ -13,6 +13,8 @@ namespace Snowflake.Data.Core abstract class SFBaseResultSet { + internal abstract ResultFormat ResultFormat { get; } + internal SFStatement sfStatement; internal SFResultSetMetaData sfResultSetMetaData; @@ -33,8 +35,6 @@ abstract class SFBaseResultSet internal abstract bool HasRows(); - internal abstract UTF8Buffer getObjectInternal(int columnIndex); - /// /// Move cursor back one row. /// @@ -44,43 +44,41 @@ abstract class SFBaseResultSet protected SFBaseResultSet() { } + + internal abstract bool IsDBNull(int ordinal); - internal T GetValue(int columnIndex) - { - UTF8Buffer val = getObjectInternal(columnIndex); - var types = sfResultSetMetaData.GetTypesByIndex(columnIndex); - return (T)SFDataConverter.ConvertToCSharpVal(val, types.Item1, typeof(T)); - } + internal abstract object GetValue(int ordinal); - internal string GetString(int columnIndex) - { - var type = sfResultSetMetaData.getColumnTypeByIndex(columnIndex); - switch (type) - { - case SFDataType.DATE: - var val = GetValue(columnIndex); - if (val == DBNull.Value) - return null; - return SFDataConverter.toDateString((DateTime)val, - sfResultSetMetaData.dateOutputFormat); - //TODO: Implement SqlFormat for timestamp type, aka parsing format specified by user and format the value - default: - return getObjectInternal(columnIndex).SafeToString(); - } - } + internal abstract bool GetBoolean(int ordinal); - internal object GetValue(int columnIndex) - { - UTF8Buffer val = getObjectInternal(columnIndex); - var types = sfResultSetMetaData.GetTypesByIndex(columnIndex); - return SFDataConverter.ConvertToCSharpVal(val, types.Item1, types.Item2); - } + internal abstract byte GetByte(int ordinal); - internal bool IsDBNull(int ordinal) - { - return (null == getObjectInternal(ordinal)); - } + internal abstract long GetBytes(int ordinal, long dataOffset, byte[] buffer, int bufferOffset, int length); + + internal abstract char GetChar(int ordinal); + + internal abstract long GetChars(int ordinal, long dataOffset, char[] buffer, int bufferOffset, int length); + internal abstract DateTime GetDateTime(int ordinal); + + internal abstract TimeSpan GetTimeSpan(int ordinal); + + internal abstract decimal GetDecimal(int ordinal); + + internal abstract double GetDouble(int ordinal); + + internal abstract float GetFloat(int ordinal); + + internal abstract Guid GetGuid(int ordinal); + + internal abstract short GetInt16(int ordinal); + + internal abstract int GetInt32(int ordinal); + + internal abstract long GetInt64(int ordinal); + + internal abstract string GetString(int ordinal); + internal void close() { isClosed = true; @@ -89,9 +87,13 @@ internal void close() internal void ThrowIfClosed() { if (isClosed) - { throw new SnowflakeDbException(SFError.DATA_READER_ALREADY_CLOSED); - } + } + + internal void ThrowIfOutOfBounds(int ordinal) + { + if (ordinal < 0 || ordinal >= columnCount) + throw new SnowflakeDbException(SFError.COLUMN_INDEX_OUT_OF_BOUND, ordinal); } } diff --git a/Snowflake.Data/Core/SFBlockingChunkDownloader.cs b/Snowflake.Data/Core/SFBlockingChunkDownloader.cs index 3049e5ea5..b3dae7928 100755 --- a/Snowflake.Data/Core/SFBlockingChunkDownloader.cs +++ b/Snowflake.Data/Core/SFBlockingChunkDownloader.cs @@ -161,7 +161,7 @@ private void ParseStreamIntoChunk(Stream content, BaseResultChunk resultChunk) Stream concatStream = new ConcatenatedStream(new Stream[3] { openBracket, content, closeBracket}); - IChunkParser parser = ChunkParserFactory.Instance.GetParser(resultChunk.Format, concatStream); + IChunkParser parser = ChunkParserFactory.Instance.GetParser(resultChunk.ResultFormat, concatStream); parser.ParseChunk(resultChunk); } } diff --git a/Snowflake.Data/Core/SFBlockingChunkDownloaderV3.cs b/Snowflake.Data/Core/SFBlockingChunkDownloaderV3.cs index 32c19caab..33451a0da 100755 --- a/Snowflake.Data/Core/SFBlockingChunkDownloaderV3.cs +++ b/Snowflake.Data/Core/SFBlockingChunkDownloaderV3.cs @@ -217,7 +217,7 @@ private async Task DownloadChunkAsync(DownloadContextV3 downloa private async Task ParseStreamIntoChunk(Stream content, BaseResultChunk resultChunk) { - IChunkParser parser = ChunkParserFactory.Instance.GetParser(resultChunk.Format, content); + IChunkParser parser = ChunkParserFactory.Instance.GetParser(resultChunk.ResultFormat, content); await parser.ParseChunk(resultChunk); } } diff --git a/Snowflake.Data/Core/SFChunkDownloaderV2.cs b/Snowflake.Data/Core/SFChunkDownloaderV2.cs index b1400e271..f654f9cd0 100755 --- a/Snowflake.Data/Core/SFChunkDownloaderV2.cs +++ b/Snowflake.Data/Core/SFChunkDownloaderV2.cs @@ -169,7 +169,7 @@ private static void ParseStreamIntoChunk(Stream content, BaseResultChunk resultC Stream concatStream = new ConcatenatedStream(new Stream[3] { openBracket, content, closeBracket}); - IChunkParser parser = ChunkParserFactory.Instance.GetParser(resultChunk.Format, concatStream); + IChunkParser parser = ChunkParserFactory.Instance.GetParser(resultChunk.ResultFormat, concatStream); parser.ParseChunk(resultChunk); } } diff --git a/Snowflake.Data/Core/SFDataConverter.cs b/Snowflake.Data/Core/SFDataConverter.cs index a01429567..1159016af 100755 --- a/Snowflake.Data/Core/SFDataConverter.cs +++ b/Snowflake.Data/Core/SFDataConverter.cs @@ -18,7 +18,7 @@ public enum SFDataType static class SFDataConverter { - private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Unspecified); + internal static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); internal static object ConvertToCSharpVal(UTF8Buffer srcVal, SFDataType srcType, Type destType) { diff --git a/Snowflake.Data/Core/SFMultiStatementsResultSet.cs b/Snowflake.Data/Core/SFMultiStatementsResultSet.cs index 735d912b4..c811deb8b 100644 --- a/Snowflake.Data/Core/SFMultiStatementsResultSet.cs +++ b/Snowflake.Data/Core/SFMultiStatementsResultSet.cs @@ -13,6 +13,8 @@ namespace Snowflake.Data.Core { class SFMultiStatementsResultSet : SFBaseResultSet { + internal override ResultFormat ResultFormat => curResultSet.ResultFormat; + private static readonly SFLogger Logger = SFLoggerFactory.GetLogger(); private string[] resultIds; @@ -107,11 +109,6 @@ internal override bool Rewind() return curResultSet.Rewind(); } - internal override UTF8Buffer getObjectInternal(int columnIndex) - { - return curResultSet.getObjectInternal(columnIndex); - } - private void updateSessionStatus(QueryExecResponseData responseData) { SFSession session = this.sfStatement.SfSession; @@ -135,5 +132,90 @@ private void updateResultMetadata() sfResultSetMetaData = null; } } + + internal override bool IsDBNull(int ordinal) + { + return curResultSet.IsDBNull(ordinal); + } + + internal override object GetValue(int ordinal) + { + return curResultSet.GetValue(ordinal); + } + + internal override bool GetBoolean(int ordinal) + { + return curResultSet.GetBoolean(ordinal); + } + + internal override byte GetByte(int ordinal) + { + return curResultSet.GetByte(ordinal); + } + + internal override long GetBytes(int ordinal, long dataOffset, byte[] buffer, int bufferOffset, int length) + { + return curResultSet.GetBytes(ordinal, dataOffset, buffer, bufferOffset, length); + } + + internal override char GetChar(int ordinal) + { + return curResultSet.GetChar(ordinal); + } + + internal override long GetChars(int ordinal, long dataOffset, char[] buffer, int bufferOffset, int length) + { + return curResultSet.GetChars(ordinal, dataOffset, buffer, bufferOffset, length); + } + + internal override DateTime GetDateTime(int ordinal) + { + return curResultSet.GetDateTime(ordinal); + } + + internal override TimeSpan GetTimeSpan(int ordinal) + { + return curResultSet.GetTimeSpan(ordinal); + } + + internal override decimal GetDecimal(int ordinal) + { + return curResultSet.GetDecimal(ordinal); + } + + internal override double GetDouble(int ordinal) + { + return curResultSet.GetDouble(ordinal); + } + + internal override float GetFloat(int ordinal) + { + return curResultSet.GetFloat(ordinal); + } + + internal override Guid GetGuid(int ordinal) + { + return curResultSet.GetGuid(ordinal); + } + + internal override short GetInt16(int ordinal) + { + return curResultSet.GetInt16(ordinal); + } + + internal override int GetInt32(int ordinal) + { + return curResultSet.GetInt32(ordinal); + } + + internal override long GetInt64(int ordinal) + { + return curResultSet.GetInt64(ordinal); + } + + internal override string GetString(int ordinal) + { + return curResultSet.GetString(ordinal); + } } } diff --git a/Snowflake.Data/Core/SFResultChunk.cs b/Snowflake.Data/Core/SFResultChunk.cs index 05aa27b63..0917bd8cd 100755 --- a/Snowflake.Data/Core/SFResultChunk.cs +++ b/Snowflake.Data/Core/SFResultChunk.cs @@ -2,13 +2,14 @@ * Copyright (c) 2012-2019 Snowflake Computing Inc. All rights reserved. */ +using System; using System.Text; namespace Snowflake.Data.Core { internal class SFResultChunk : BaseResultChunk { - internal override ResultFormat Format => ResultFormat.JSON; + internal override ResultFormat ResultFormat => ResultFormat.JSON; private int _currentRowIndex = -1; @@ -27,6 +28,7 @@ public SFResultChunk(string url, int rowCount, int columnCount, int index) ChunkIndex = index; } + [Obsolete("ExtractCell with rowIndex is deprecated", false)] public override UTF8Buffer ExtractCell(int rowIndex, int columnIndex) { _currentRowIndex = rowIndex; diff --git a/Snowflake.Data/Core/SFResultSet.cs b/Snowflake.Data/Core/SFResultSet.cs index 2e20453ba..764ce0907 100755 --- a/Snowflake.Data/Core/SFResultSet.cs +++ b/Snowflake.Data/Core/SFResultSet.cs @@ -2,6 +2,7 @@ * Copyright (c) 2012-2019 Snowflake Computing Inc. All rights reserved. */ +using System; using System.Threading; using System.Threading.Tasks; using Snowflake.Data.Log; @@ -12,6 +13,8 @@ namespace Snowflake.Data.Core { class SFResultSet : SFBaseResultSet { + internal override ResultFormat ResultFormat => ResultFormat.JSON; + private static readonly SFLogger s_logger = SFLoggerFactory.GetLogger(); private readonly int _totalChunkCount; @@ -172,16 +175,12 @@ internal override bool Rewind() return _currentChunk.Rewind(); } - internal override UTF8Buffer getObjectInternal(int columnIndex) + internal UTF8Buffer GetObjectInternal(int ordinal) { ThrowIfClosed(); + ThrowIfOutOfBounds(ordinal); - if (columnIndex < 0 || columnIndex >= columnCount) - { - throw new SnowflakeDbException(SFError.COLUMN_INDEX_OUT_OF_BOUND, columnIndex); - } - - return _currentChunk.ExtractCell(columnIndex); + return _currentChunk.ExtractCell(ordinal); } private void UpdateSessionStatus(QueryExecResponseData responseData) @@ -191,5 +190,182 @@ private void UpdateSessionStatus(QueryExecResponseData responseData) session.UpdateSessionParameterMap(responseData.parameters); session.UpdateQueryContextCache(responseData.QueryContext); } + + internal override bool IsDBNull(int ordinal) + { + return (null == GetObjectInternal(ordinal)); + } + + internal override bool GetBoolean(int ordinal) + { + return GetValue(ordinal); + } + + internal override byte GetByte(int ordinal) + { + return GetValue(ordinal); + } + + internal override long GetBytes(int ordinal, long dataOffset, byte[] buffer, int bufferOffset, int length) + { + return ReadSubset(ordinal, dataOffset, buffer, bufferOffset, length); + } + + internal override char GetChar(int ordinal) + { + string val = GetString(ordinal); + return val[0]; + } + + internal override long GetChars(int ordinal, long dataOffset, char[] buffer, int bufferOffset, int length) + { + return ReadSubset(ordinal, dataOffset, buffer, bufferOffset, length); + } + + internal override DateTime GetDateTime(int ordinal) + { + return GetValue(ordinal); + } + + internal override TimeSpan GetTimeSpan(int ordinal) + { + return GetValue(ordinal); + } + + internal override decimal GetDecimal(int ordinal) + { + return GetValue(ordinal); + } + + internal override double GetDouble(int ordinal) + { + return GetValue(ordinal); + } + + internal override float GetFloat(int ordinal) + { + return GetValue(ordinal); + } + + internal override Guid GetGuid(int ordinal) + { + return GetValue(ordinal); + } + + internal override short GetInt16(int ordinal) + { + return GetValue(ordinal); + } + + internal override int GetInt32(int ordinal) + { + return GetValue(ordinal); + } + + internal override long GetInt64(int ordinal) + { + return GetValue(ordinal); + } + + internal override string GetString(int ordinal) + { + ThrowIfOutOfBounds(ordinal); + + var type = sfResultSetMetaData.GetColumnTypeByIndex(ordinal); + switch (type) + { + case SFDataType.DATE: + var val = GetValue(ordinal); + if (val == DBNull.Value) + return null; + return SFDataConverter.toDateString((DateTime)val, sfResultSetMetaData.dateOutputFormat); + + default: + return GetObjectInternal(ordinal).SafeToString(); + } + } + + internal override object GetValue(int ordinal) + { + UTF8Buffer val = GetObjectInternal(ordinal); + var types = sfResultSetMetaData.GetTypesByIndex(ordinal); + return SFDataConverter.ConvertToCSharpVal(val, types.Item1, types.Item2); + } + + private T GetValue(int ordinal) + { + UTF8Buffer val = GetObjectInternal(ordinal); + var types = sfResultSetMetaData.GetTypesByIndex(ordinal); + return (T)SFDataConverter.ConvertToCSharpVal(val, types.Item1, typeof(T)); + } + + // + // Summary: + // Reads a subset of data starting at location indicated by dataOffset into the buffer, + // starting at the location indicated by bufferOffset. + // + // Parameters: + // ordinal: + // The zero-based column ordinal. + // + // dataOffset: + // The index within the data from which to begin the read operation. + // + // buffer: + // The buffer into which to copy the data. + // + // bufferOffset: + // The index with the buffer to which the data will be copied. + // + // length: + // The maximum number of elements to read. + // + // Returns: + // The actual number of elements read. + private long ReadSubset(int ordinal, long dataOffset, T[] buffer, int bufferOffset, int length) where T : struct + { + if (dataOffset < 0) + { + throw new ArgumentOutOfRangeException("dataOffset", "Non negative number is required."); + } + + if (bufferOffset < 0) + { + throw new ArgumentOutOfRangeException("bufferOffset", "Non negative number is required."); + } + + if ((null != buffer) && (bufferOffset > buffer.Length)) + { + throw new System.ArgumentException("Destination buffer is not long enough. " + + "Check the buffer offset, length, and the buffer's lower bounds.", "buffer"); + } + + T[] data = GetValue(ordinal); + + // https://docs.microsoft.com/en-us/dotnet/api/system.data.idatarecord.getbytes?view=net-5.0#remarks + // If you pass a buffer that is null, GetBytes returns the length of the row in bytes. + // https://docs.microsoft.com/en-us/dotnet/api/system.data.idatarecord.getchars?view=net-5.0#remarks + // If you pass a buffer that is null, GetChars returns the length of the field in characters. + if (null == buffer) + { + return data.Length; + } + + if (dataOffset > data.Length) + { + throw new System.ArgumentException("Source data is not long enough. " + + "Check the data offset, length, and the data's lower bounds." ,"dataOffset"); + } + else + { + // How much data is available after the offset + long dataLength = data.Length - dataOffset; + // How much data to read + long elementsRead = Math.Min(length, dataLength); + Array.Copy(data, dataOffset, buffer, bufferOffset, elementsRead); + + return elementsRead; + } + } } } diff --git a/Snowflake.Data/Core/SFResultSetMetaData.cs b/Snowflake.Data/Core/SFResultSetMetaData.cs index 1045109f4..7fd510bf8 100755 --- a/Snowflake.Data/Core/SFResultSetMetaData.cs +++ b/Snowflake.Data/Core/SFResultSetMetaData.cs @@ -41,7 +41,7 @@ internal SFResultSetMetaData(QueryExecResponseData queryExecResponseData) { rowTypes = queryExecResponseData.rowType; columnCount = rowTypes.Count; - statementType = findStatementTypeById(queryExecResponseData.statementTypeId); + statementType = FindStatementTypeById(queryExecResponseData.statementTypeId); columnTypes = InitColumnTypes(); foreach (NameValueParameter parameter in queryExecResponseData.parameters) @@ -62,7 +62,7 @@ internal SFResultSetMetaData(PutGetResponseData putGetResponseData) { rowTypes = putGetResponseData.rowType; columnCount = rowTypes.Count; - statementType = findStatementTypeById(putGetResponseData.statementTypeId); + statementType = FindStatementTypeById(putGetResponseData.statementTypeId); columnTypes = InitColumnTypes(); } @@ -83,10 +83,9 @@ private List> InitColumnTypes() /// /// /// index of column given a name, -1 if no column names are found - internal int getColumnIndexByName(string targetColumnName) + internal int GetColumnIndexByName(string targetColumnName) { - int resultIndex; - if (columnNameToIndexCache.TryGetValue(targetColumnName, out resultIndex)) + if (columnNameToIndexCache.TryGetValue(targetColumnName, out var resultIndex)) { return resultIndex; } @@ -95,9 +94,9 @@ internal int getColumnIndexByName(string targetColumnName) int indexCounter = 0; foreach (ExecResponseRowType rowType in rowTypes) { - if (String.Compare(rowType.name, targetColumnName, false ) == 0 ) + if (String.Compare(rowType.name, targetColumnName, false) == 0 ) { - logger.Info($"Found colun name {targetColumnName} under index {indexCounter}"); + logger.Info($"Found column name {targetColumnName} under index {indexCounter}"); columnNameToIndexCache[targetColumnName] = indexCounter; return indexCounter; } @@ -107,26 +106,21 @@ internal int getColumnIndexByName(string targetColumnName) return -1; } - internal SFDataType getColumnTypeByIndex(int targetIndex) + internal SFDataType GetColumnTypeByIndex(int targetIndex) { - if (targetIndex < 0 || targetIndex >= columnCount) - { - throw new SnowflakeDbException(SFError.COLUMN_INDEX_OUT_OF_BOUND, targetIndex); - } - return columnTypes[targetIndex].Item1; } internal Tuple GetTypesByIndex(int targetIndex) { - if (targetIndex < 0 || targetIndex >= columnCount) - { - throw new SnowflakeDbException(SFError.COLUMN_INDEX_OUT_OF_BOUND, targetIndex); - } - return columnTypes[targetIndex]; } + internal long GetScaleByIndex(int targetIndex) + { + return rowTypes[targetIndex].scale; + } + private SFDataType GetSFDataType(string type) { SFDataType rslt; @@ -167,33 +161,17 @@ private Type GetNativeTypeForColumn(SFDataType sfType, ExecResponseRowType col) } } - internal Type getCSharpTypeByIndex(int targetIndex) + internal Type GetCSharpTypeByIndex(int targetIndex) { - if (targetIndex < 0 || targetIndex >= columnCount) - { - throw new SnowflakeDbException(SFError.COLUMN_INDEX_OUT_OF_BOUND, targetIndex); - } - - SFDataType sfType = getColumnTypeByIndex(targetIndex); - return GetNativeTypeForColumn(sfType, rowTypes[targetIndex]); + return columnTypes[targetIndex].Item2; } - internal string getColumnNameByIndex(int targetIndex) + internal string GetColumnNameByIndex(int targetIndex) { - if (targetIndex < 0 || targetIndex >= columnCount) - { - throw new SnowflakeDbException(SFError.COLUMN_INDEX_OUT_OF_BOUND, targetIndex); - } - return rowTypes[targetIndex].name; } - internal DataTable toDataTable() - { - return null; - } - - private SFStatementType findStatementTypeById(long id) + private SFStatementType FindStatementTypeById(long id) { foreach (SFStatementType type in Enum.GetValues(typeof(SFStatementType))) { diff --git a/Snowflake.Data/Core/SFReusableChunk.cs b/Snowflake.Data/Core/SFReusableChunk.cs index 8e36907a0..06ea7cef3 100755 --- a/Snowflake.Data/Core/SFReusableChunk.cs +++ b/Snowflake.Data/Core/SFReusableChunk.cs @@ -10,7 +10,7 @@ namespace Snowflake.Data.Core { class SFReusableChunk : BaseResultChunk { - internal override ResultFormat Format => ResultFormat.JSON; + internal override ResultFormat ResultFormat => ResultFormat.JSON; private readonly BlockResultData data; @@ -34,6 +34,7 @@ internal override void ResetForRetry() data.ResetForRetry(); } + [Obsolete("ExtractCell with rowIndex is deprecated", false)] public override UTF8Buffer ExtractCell(int rowIndex, int columnIndex) { _currentRowIndex = rowIndex;