From 3dd32c0e58e900c2be6adb9da519f6a472d0cd6b Mon Sep 17 00:00:00 2001 From: Krzysztof Nozderko Date: Fri, 7 Jun 2024 09:47:58 +0000 Subject: [PATCH] SNOW-981350 Structured types for json format - work in progress --- Snowflake.Data.Tests/Client/Address.cs | 98 ++++++++ .../IntegrationTests/StructuredTypesIT.cs | 212 ++++++++++++++++++ Snowflake.Data.Tests/SFBaseTest.cs | 35 +-- .../Client/SnowflakeDbDataReader.cs | 42 +++- .../JsonToStructuredTypeConverter.cs | 171 ++++++++++++++ .../Core/Converter/TypePropertyExtracotor.cs | 150 +++++++++++++ Snowflake.Data/Core/HttpUtil.cs | 13 +- Snowflake.Data/Core/RestResponse.cs | 46 +++- Snowflake.Data/Core/SFDataConverter.cs | 36 +-- Snowflake.Data/Core/SFResultSetMetaData.cs | 26 ++- 10 files changed, 768 insertions(+), 61 deletions(-) create mode 100644 Snowflake.Data.Tests/Client/Address.cs create mode 100644 Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs create mode 100644 Snowflake.Data/Core/Converter/JsonToStructuredTypeConverter.cs create mode 100644 Snowflake.Data/Core/Converter/TypePropertyExtracotor.cs diff --git a/Snowflake.Data.Tests/Client/Address.cs b/Snowflake.Data.Tests/Client/Address.cs new file mode 100644 index 000000000..cb0c4e0a2 --- /dev/null +++ b/Snowflake.Data.Tests/Client/Address.cs @@ -0,0 +1,98 @@ +using System.Collections.Generic; + +namespace Snowflake.Data.Tests.IntegrationTests +{ + public class Address + { + public string city { get; set; } + public string state { get; set; } + public Zip zip { get; set; } + + public Address() + { + } + + public Address(string city, string state, Zip zip) + { + this.city = city; + this.state = state; + this.zip = zip; + } + } + + public class Zip + { + public string prefix { get; set; } + public string postfix { get; set; } + + public Zip() + { + } + + public Zip(string prefix, string postfix) + { + this.prefix = prefix; + this.postfix = postfix; + } + } + + public class Identity + { + public string Name { get; set; } + + public Identity() + { + } + + public Identity(string name) + { + Name = name; + } + + protected bool Equals(Identity other) + { + return Name == other.Name; + } + + public override bool Equals(object obj) + { + if (ReferenceEquals(null, obj)) return false; + if (ReferenceEquals(this, obj)) return true; + if (obj.GetType() != this.GetType()) return false; + return Equals((Identity)obj); + } + + public override int GetHashCode() + { + return (Name != null ? Name.GetHashCode() : 0); + } + } + + public class Grades + { + public string[] Names { get; set; } + + public Grades() + { + } + + public Grades(string[] names) + { + Names = names; + } + } + + public class GradesWithList + { + public List Names { get; set; } + + public GradesWithList() + { + } + + public GradesWithList(List names) + { + Names = names; + } + } +} diff --git a/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs b/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs new file mode 100644 index 000000000..01f022dee --- /dev/null +++ b/Snowflake.Data.Tests/IntegrationTests/StructuredTypesIT.cs @@ -0,0 +1,212 @@ +using System.Collections.Generic; +using NUnit.Framework; +using Snowflake.Data.Client; + +namespace Snowflake.Data.Tests.IntegrationTests +{ + [TestFixture] + public class StructuredTypesIT: SFBaseTest + { + private static string _tableName = "structured_types_tests"; + + [Test] + public void TestInsertStructuredTypeObject() + { + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + // arrange + connection.Open(); + CreateOrReplaceTable(connection, _tableName, new List {"address OBJECT(city VARCHAR, state VARCHAR)"}); + using (var command = connection.CreateCommand()) + { + var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA')::OBJECT(city VARCHAR, state VARCHAR)"; + command.CommandText = $"INSERT INTO {_tableName} SELECT {addressAsSFString}"; + command.ExecuteNonQuery(); + command.CommandText = $"SELECT * FROM {_tableName}"; + + // act + var reader = command.ExecuteReader(); + + // assert + Assert.IsTrue(reader.Read()); + } + } + } + + [Test] + public void TestSelectStructuredTypeObject() + { + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + // arrange + connection.Open(); + using (var command = connection.CreateCommand()) + { + var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA')::OBJECT(city VARCHAR, state VARCHAR)"; + command.CommandText = $"SELECT {addressAsSFString}"; + + // act + var reader = (SnowflakeDbDataReader) command.ExecuteReader(); + + // assert + Assert.IsTrue(reader.Read()); + var address = reader.GetObject
(0); + Assert.AreEqual("San Mateo", address.city); + Assert.AreEqual("CA", address.state); + Assert.IsNull(address.zip); + } + } + } + + [Test] + [TestCase(StructureTypeConstructionMethod.PROPERTIES_NAMES)] + [TestCase(StructureTypeConstructionMethod.PROPERTIES_ORDER)] + [TestCase(StructureTypeConstructionMethod.CONSTRUCTOR)] + public void TestSelectNestedStructuredTypeObject(StructureTypeConstructionMethod constructionMethod) + { + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + // arrange + connection.Open(); + using (var command = connection.CreateCommand()) + { + var addressAsSFString = "OBJECT_CONSTRUCT('city','San Mateo', 'state', 'CA', 'zip', OBJECT_CONSTRUCT('prefix', '00', 'postfix', '11'))::OBJECT(city VARCHAR, state VARCHAR, zip OBJECT(prefix VARCHAR, postfix VARCHAR))"; + command.CommandText = $"SELECT {addressAsSFString}"; + + // act + var reader = (SnowflakeDbDataReader) command.ExecuteReader(); + + // assert + Assert.IsTrue(reader.Read()); + var address = reader.GetObject
(0, constructionMethod); + Assert.AreEqual("San Mateo", address.city); + Assert.AreEqual("CA", address.state); + Assert.NotNull(address.zip); + Assert.AreEqual("00", address.zip.prefix); + Assert.AreEqual("11", address.zip.postfix); + } + } + } + + [Test] + public void TestSelectArray() + { + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + // arrange + connection.Open(); + using (var command = connection.CreateCommand()) + { + var arrayOfNumberSFString = "ARRAY_CONSTRUCT('a','b','c')::ARRAY(TEXT)"; + command.CommandText = $"SELECT {arrayOfNumberSFString}"; + + // act + var reader = (SnowflakeDbDataReader) command.ExecuteReader(); + + // assert + Assert.IsTrue(reader.Read()); + var array = reader.GetArray(0); + Assert.AreEqual(3, array.Length); + CollectionAssert.AreEqual(new []{ "a", "b", "c"}, array); + } + } + } + + [Test] + public void TestSelectArrayOfObjects() + { + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + // arrange + connection.Open(); + using (var command = connection.CreateCommand()) + { + var arrayOfObjects = "ARRAY_CONSTRUCT(OBJECT_CONSTRUCT('name', 'Alex'), OBJECT_CONSTRUCT('name', 'Brian'))::ARRAY(OBJECT(name VARCHAR))"; + command.CommandText = $"SELECT {arrayOfObjects}"; + + // act + var reader = (SnowflakeDbDataReader) command.ExecuteReader(); + + // assert + Assert.IsTrue(reader.Read()); + var array = reader.GetArray(0); + Assert.AreEqual(2, array.Length); + CollectionAssert.AreEqual(new []{new Identity("Alex"), new Identity("Brian")}, array); + } + } + } + + + [Test] + public void TestSelectArrayOfArrays() + { + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + // arrange + connection.Open(); + using (var command = connection.CreateCommand()) + { + var arrayOfObjects = "ARRAY_CONSTRUCT(ARRAY_CONSTRUCT('a', 'b'), ARRAY_CONSTRUCT('c', 'd'))::ARRAY(ARRAY(TEXT))"; + command.CommandText = $"SELECT {arrayOfObjects}"; + + // act + var reader = (SnowflakeDbDataReader) command.ExecuteReader(); + + // assert + Assert.IsTrue(reader.Read()); + var array = reader.GetArray(0); + Assert.AreEqual(2, array.Length); + CollectionAssert.AreEqual(new []{ new [] {"a", "b"}, new [] {"c", "d"}}, array); + } + } + } + + [Test] + public void TestSelectObjectWithArrays() + { + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + // arrange + connection.Open(); + using (var command = connection.CreateCommand()) + { + var objectWithArray = "OBJECT_CONSTRUCT('names', ARRAY_CONSTRUCT('Excellent', 'Poor'))::OBJECT(names ARRAY(TEXT))"; + command.CommandText = $"SELECT {objectWithArray}"; + + // act + var reader = (SnowflakeDbDataReader) command.ExecuteReader(); + + // assert + Assert.IsTrue(reader.Read()); + var grades = reader.GetObject(0); + Assert.NotNull(grades); + CollectionAssert.AreEqual(new [] {"Excellent", "Poor"}, grades.Names); + } + } + } + + [Test] + public void TestSelectObjectWithList() + { + using (var connection = new SnowflakeDbConnection(ConnectionString)) + { + // arrange + connection.Open(); + using (var command = connection.CreateCommand()) + { + var objectWithArray = "OBJECT_CONSTRUCT('names', ARRAY_CONSTRUCT('Excellent', 'Poor'))::OBJECT(names ARRAY(TEXT))"; + command.CommandText = $"SELECT {objectWithArray}"; + + // act + var reader = (SnowflakeDbDataReader) command.ExecuteReader(); + + // assert + Assert.IsTrue(reader.Read()); + var grades = reader.GetObject(0); + Assert.NotNull(grades); + CollectionAssert.AreEqual(new List {"Excellent", "Poor"}, grades.Names); + } + } + } + } +} diff --git a/Snowflake.Data.Tests/SFBaseTest.cs b/Snowflake.Data.Tests/SFBaseTest.cs index 6aacb94f9..8c00d818c 100755 --- a/Snowflake.Data.Tests/SFBaseTest.cs +++ b/Snowflake.Data.Tests/SFBaseTest.cs @@ -25,9 +25,9 @@ namespace Snowflake.Data.Tests using Newtonsoft.Json.Serialization; /* - * This is the base class for all tests that call blocking methods in the library - it uses MockSynchronizationContext to verify that + * This is the base class for all tests that call blocking methods in the library - it uses MockSynchronizationContext to verify that * there are no async deadlocks in the library - * + * */ [TestFixture] public class SFBaseTest : SFBaseTestAsync @@ -47,7 +47,7 @@ public static void TearDownContext() /* * This is the base class for all tests that call async methods in the library - it does not use a special SynchronizationContext - * + * */ [TestFixture] [FixtureLifeCycle(LifeCycle.InstancePerTestCase)] @@ -61,16 +61,17 @@ public class SFBaseTestAsync private const string ConnectionStringWithoutAuthFmt = "scheme={0};host={1};port={2};" + "account={3};role={4};db={5};schema={6};warehouse={7}"; + // + ";useProxy=true;proxyHost=localhost;proxyPort=8080"; private const string ConnectionStringSnowflakeAuthFmt = ";user={0};password={1};"; protected virtual string TestName => TestContext.CurrentContext.Test.MethodName; protected string TestNameWithWorker => TestName + TestContext.CurrentContext.WorkerId?.Replace("#", "_"); protected string TableName => TestNameWithWorker; - + private Stopwatch _stopwatch; private List _tablesToRemove; - + [SetUp] public void BeforeTest() { @@ -93,7 +94,7 @@ private void RemoveTables() { if (_tablesToRemove.Count == 0) return; - + using (var conn = new SnowflakeDbConnection(ConnectionString)) { conn.Open(); @@ -148,26 +149,26 @@ public SFBaseTestAsync() string.Format(ConnectionStringSnowflakeAuthFmt, testConfig.user, testConfig.password); - + protected string ConnectionStringWithInvalidUserName => ConnectionStringWithoutAuth + string.Format(ConnectionStringSnowflakeAuthFmt, "unknown", testConfig.password); protected TestConfig testConfig { get; } - + protected string ResolveHost() { return testConfig.host ?? $"{testConfig.account}.snowflakecomputing.com"; } } - + [SetUpFixture] public class TestEnvironment { - private const string ConnectionStringFmt = "scheme={0};host={1};port={2};" + + private const string ConnectionStringFmt = "scheme={0};host={1};port={2};" + "account={3};role={4};db={5};warehouse={6};user={7};password={8};"; - + public static TestConfig TestConfig { get; private set; } private static Dictionary s_testPerformance; @@ -201,7 +202,7 @@ public void Setup() var testConfigString = reader.ReadToEnd(); - // Local JSON settings to avoid using system wide settings which could be different + // Local JSON settings to avoid using system wide settings which could be different // than the default ones var jsonSettings = new JsonSerializerSettings { @@ -221,16 +222,16 @@ public void Setup() { Assert.Fail("Failed to load test configuration"); } - + ModifySchema(TestConfig.schema, SchemaAction.CREATE); } - + [OneTimeTearDown] public void Cleanup() { ModifySchema(TestConfig.schema, SchemaAction.DROP); } - + [OneTimeSetUp] public void SetupTestPerformance() { @@ -243,12 +244,12 @@ public void CreateTestTimeArtifact() var resultText = "test;time_in_ms\n"; resultText += string.Join("\n", s_testPerformance.Select(test => $"{test.Key};{Math.Round(test.Value.TotalMilliseconds,0)}")); - + var dotnetVersion = Environment.GetEnvironmentVariable("net_version"); var cloudEnv = Environment.GetEnvironmentVariable("snowflake_cloud_env"); var separator = Path.DirectorySeparatorChar; - + // We have to go up 3 times as the working directory path looks as follows: // Snowflake.Data.Tests/bin/debug/{.net_version}/ File.WriteAllText($"..{separator}..{separator}..{separator}{GetOs()}_{dotnetVersion}_{cloudEnv}_performance.csv", resultText); diff --git a/Snowflake.Data/Client/SnowflakeDbDataReader.cs b/Snowflake.Data/Client/SnowflakeDbDataReader.cs index 20ca1f7ba..9442bf219 100755 --- a/Snowflake.Data/Client/SnowflakeDbDataReader.cs +++ b/Snowflake.Data/Client/SnowflakeDbDataReader.cs @@ -10,8 +10,8 @@ using System.Threading; using System.Threading.Tasks; using Snowflake.Data.Log; -using System.Text; -using System.IO; +using Newtonsoft.Json.Linq; +using Snowflake.Data.Core.Converter; namespace Snowflake.Data.Client { @@ -30,7 +30,7 @@ public class SnowflakeDbDataReader : DbDataReader private int RecordsAffectedInternal; internal ResultFormat ResultFormat => resultSet.ResultFormat; - + internal SnowflakeDbDataReader(SnowflakeDbCommand command, SFBaseResultSet resultSet) { this.dbCommand = command; @@ -99,7 +99,7 @@ public string GetQueryId() { return resultSet.queryId; } - + private DataTable PopulateSchemaTable(SFBaseResultSet resultSet) { var table = new DataTable("SchemaTable"); @@ -136,7 +136,7 @@ private DataTable PopulateSchemaTable(SFBaseResultSet resultSet) return table; } - + public override bool GetBoolean(int ordinal) { return resultSet.GetBoolean(ordinal); @@ -255,6 +255,31 @@ public override int GetValues(object[] values) return count; } + public T GetObject(int ordinal, StructureTypeConstructionMethod constructionMethod = StructureTypeConstructionMethod.PROPERTIES_ORDER) + where T : class, new() + { + var rowType = resultSet.sfResultSetMetaData.rowTypes[ordinal]; + var fields = rowType.fields; + if (fields == null || fields.Count == 0) + { + return (T) GetValue(ordinal); + } + var json = JObject.Parse(GetString(ordinal)); + return JsonToStructuredTypeConverter.Convert(rowType.type, fields, json, constructionMethod); + } + + public T[] GetArray(int ordinal, StructureTypeConstructionMethod constructionMethod = StructureTypeConstructionMethod.PROPERTIES_ORDER) + { + var rowType = resultSet.sfResultSetMetaData.rowTypes[ordinal]; + var fields = rowType.fields; + if (fields == null || fields.Count == 0) + { + return (T[]) GetValue(ordinal); + } + var json = JArray.Parse(GetString(ordinal)); + return JsonToStructuredTypeConverter.ConvertArray(rowType.type, fields, json, constructionMethod); + } + public override bool IsDBNull(int ordinal) { return resultSet.IsDBNull(ordinal); @@ -302,4 +327,11 @@ public override void Close() } } + + public enum StructureTypeConstructionMethod + { + PROPERTIES_ORDER, + PROPERTIES_NAMES, + CONSTRUCTOR + } } diff --git a/Snowflake.Data/Core/Converter/JsonToStructuredTypeConverter.cs b/Snowflake.Data/Core/Converter/JsonToStructuredTypeConverter.cs new file mode 100644 index 000000000..acbf6d897 --- /dev/null +++ b/Snowflake.Data/Core/Converter/JsonToStructuredTypeConverter.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using Newtonsoft.Json.Linq; +using Snowflake.Data.Client; + +namespace Snowflake.Data.Core.Converter +{ + internal class JsonToStructuredTypeConverter + { + public static T Convert(string sourceTypeName, List fields, JObject value, + StructureTypeConstructionMethod constructionMethod) + { + var type = typeof(T); + if (SFDataType.OBJECT.ToString().Equals(sourceTypeName, StringComparison.OrdinalIgnoreCase)) + { + return (T) ConvertToObject(type, fields, value, constructionMethod); + } + + throw new Exception("Case not supported"); + } + + public static T[] ConvertArray(string sourceTypeName, List fields, JArray value, + StructureTypeConstructionMethod constructionMethod) + { + var type = typeof(T[]); + var elementType = typeof(T); + if (SFDataType.ARRAY.ToString().Equals(sourceTypeName, StringComparison.OrdinalIgnoreCase)) + { + return (T[]) ConvertToArray(type, elementType, fields, value, constructionMethod); + } + + throw new Exception("Case not supported"); + } + + private static object ConvertToObject(Type type, List fields, JToken json, StructureTypeConstructionMethod constructionMethod) + { + if (json.Type == JTokenType.Null || json.Type == JTokenType.Undefined) + { + return null; + } + if (json.Type != JTokenType.Object) + { + throw new Exception("Json is not an object"); + } + var jsonObject = (JObject)json; + var objectBuilder = ObjectBuilderFactory.Create(type, fields?.Count ?? 0, constructionMethod); + using (var jsonEnumerator = jsonObject.GetEnumerator()) + { + + var metadataIterator = fields.GetEnumerator(); + while (jsonEnumerator.MoveNext() && metadataIterator.MoveNext()) + { + var jsonPropertyWithValue = jsonEnumerator.Current; + var fieldMetadata = metadataIterator.Current; + var key = jsonPropertyWithValue.Key; + var fieldValue = jsonPropertyWithValue.Value; + if (IsTextMetadata(fieldMetadata)) + { + var stringValue = fieldValue.Value(); + var fieldType = objectBuilder.MoveNext(key); + objectBuilder.BuildPart(stringValue); + } else if (IsObjectMetadata(fieldMetadata)) + { + var fieldType = objectBuilder.MoveNext(key); + var objectValue = ConvertToObject(fieldType, fieldMetadata.fields, fieldValue, constructionMethod); + objectBuilder.BuildPart(objectValue); + } + else if (IsArrayMetadata(fieldMetadata)) + { + var fieldType = objectBuilder.MoveNext(key); + var nestedType = GetNestedType(fieldType); + var arrayValue = ConvertToArray(fieldType, nestedType, fieldMetadata.fields, fieldValue, constructionMethod); + objectBuilder.BuildPart(arrayValue); + } + else + { + throw new Exception("Case not implemented yet"); + } + } + } + return objectBuilder.Build(); + } + + private static object ConvertToArray(Type type, Type elementType, List fields, JToken json, StructureTypeConstructionMethod constructionMethod) + { + if (json.Type == JTokenType.Null || json.Type == JTokenType.Undefined) + { + return null; + } + if (json.Type != JTokenType.Array) + { + throw new Exception("Json is not an array"); + } + var jsonArray = (JArray)json; + var arrayType = MakeArrayType(type, elementType); + var result = (object[]) Activator.CreateInstance(arrayType, jsonArray.Count); + var elementMetadata = fields[0]; + for (var i = 0; i < jsonArray.Count; i++) + { + if (IsObjectMetadata(elementMetadata)) + { + result[i] = ConvertToObject(elementType, elementMetadata.fields, jsonArray[i], constructionMethod); + } + else if (IsTextMetadata(elementMetadata)) + { + result[i] = jsonArray[i].Value(); + } + else if (IsArrayMetadata(elementMetadata)) + { + var nestedType = elementType.GetElementType(); + result[i] = ConvertToArray(elementType, nestedType, elementMetadata.fields, jsonArray[i], constructionMethod); + } + else + { + throw new Exception("Case not implemented yet"); + } + } + if (type != arrayType) + { + var list = (IList) Activator.CreateInstance(type); + for (int i = 0; i < result.Length; i++) + { + list.Add(result[i]); + } + return list; + } + + return result; + } + + private static Type GetNestedType(Type type) + { + if (type.IsArray) + { + return type.GetElementType(); + } + if (IsListType(type)) + { + return type.GenericTypeArguments[0]; + } + throw new Exception("neither array nor list"); + } + + private static Type MakeArrayType(Type type, Type elementType) + { + if (type.IsArray) + { + return type; + } + if (IsListType(type)) + { + return elementType.MakeArrayType(); + } + throw new Exception("Neither array nor list"); + } + + // JValue, JObject, JArray ... are elements of JArray + + private static bool IsListType(Type type) => type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>); + + private static bool IsObjectMetadata(FieldMetadata fieldMetadata) => + SFDataType.OBJECT.ToString().Equals(fieldMetadata.type, StringComparison.OrdinalIgnoreCase); + + private static bool IsTextMetadata(FieldMetadata fieldMetadata) => + SFDataType.TEXT.ToString().Equals(fieldMetadata.type, StringComparison.OrdinalIgnoreCase); + + private static bool IsArrayMetadata(FieldMetadata fieldMetadata) => + SFDataType.ARRAY.ToString().Equals(fieldMetadata.type, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/Snowflake.Data/Core/Converter/TypePropertyExtracotor.cs b/Snowflake.Data/Core/Converter/TypePropertyExtracotor.cs new file mode 100644 index 000000000..05d87961e --- /dev/null +++ b/Snowflake.Data/Core/Converter/TypePropertyExtracotor.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Snowflake.Data.Client; + +namespace Snowflake.Data.Core.Converter +{ + internal static class ObjectBuilderFactory + { + public static IObjectBuilder Create(Type type, int fieldsCount, StructureTypeConstructionMethod constructionMethod) + { + if (constructionMethod == StructureTypeConstructionMethod.PROPERTIES_NAMES) + { + return new ObjectBuilderByPropertyNames(type); + } + if (constructionMethod == StructureTypeConstructionMethod.PROPERTIES_ORDER) + { + return new ObjectBuilderByPropertyOrder(type); + } + if (constructionMethod == StructureTypeConstructionMethod.CONSTRUCTOR) + { + return new ObjectBuilderByConstructor(type, fieldsCount); + } + throw new Exception("Unknown construction method"); + } + } + + internal interface IObjectBuilder + { + void BuildPart(object value); + + Type MoveNext(string propertyName); + + object Build(); + } + + internal class ObjectBuilderByPropertyNames : IObjectBuilder + { + private Type _type; + private List> _result; + private PropertyInfo _currentProperty; + + public ObjectBuilderByPropertyNames(Type type) + { + _type = type; + _result = new List>(); + } + + public void BuildPart(object value) + { + _result.Add(Tuple.Create(_currentProperty, value)); + } + + public Type MoveNext(string propertyName) + { + _currentProperty = _type.GetProperty(propertyName); + if (_currentProperty == null) + { + throw new Exception($"Could not find property: {propertyName}"); + } + return _currentProperty.PropertyType; + } + + public object Build() + { + var result = Activator.CreateInstance(_type); + _result.ForEach(p => p.Item1.SetValue(result, p.Item2, null)); + return result; + } + } + + internal class ObjectBuilderByPropertyOrder : IObjectBuilder + { + private Type _type; + private PropertyInfo[] _properties; + private int _index; + private List> _result; + private PropertyInfo _currentProperty; + + public ObjectBuilderByPropertyOrder(Type type) + { + _type = type; + _properties = type.GetProperties(); + _index = -1; + _result = new List>(); + } + + public void BuildPart(object value) + { + _result.Add(Tuple.Create(_currentProperty, value)); + } + + public Type MoveNext(string propertyName) + { + _index++; + _currentProperty = _properties[_index]; + return _currentProperty.PropertyType; + } + + public object Build() + { + var result = Activator.CreateInstance(_type); + _result.ForEach(p => p.Item1.SetValue(result, p.Item2, null)); + return result; + } + } + + internal class ObjectBuilderByConstructor : IObjectBuilder + { + private Type _type; + private List _result; + private Type[] _parameters; + private int _index; + + public ObjectBuilderByConstructor(Type type, int fieldsCount) + { + _type = type; + var matchingConstructors = type.GetConstructors() + .Where(c => c.GetParameters().Length == fieldsCount) + .ToList(); + if (matchingConstructors.Count == 0) + { + throw new Exception($"Proper constructor not found for type: {type}"); + } + var constructor = matchingConstructors.First(); + _parameters = constructor.GetParameters().Select(p => p.ParameterType).ToArray(); + _index = -1; + _result = new List(); + } + + public Type MoveNext(string propertyName) + { + _index++; + return _parameters[_index]; + } + + public void BuildPart(object value) + { + _result.Add(value); + } + + public object Build() + { + object[] parameters = _result.ToArray(); + return Activator.CreateInstance(_type, parameters); + } + + } +} diff --git a/Snowflake.Data/Core/HttpUtil.cs b/Snowflake.Data/Core/HttpUtil.cs index 531e76fd7..ecaa8c4c5 100755 --- a/Snowflake.Data/Core/HttpUtil.cs +++ b/Snowflake.Data/Core/HttpUtil.cs @@ -87,7 +87,7 @@ public sealed class HttpUtil private HttpUtil() { - // This value is used by AWS SDK and can cause deadlock, + // This value is used by AWS SDK and can cause deadlock, // so we need to increase the default value of 2 // See: https://github.com/aws/aws-sdk-net/issues/152 ServicePointManager.DefaultConnectionLimit = 50; @@ -142,6 +142,7 @@ internal HttpMessageHandler SetupCustomHttpHandler(HttpClientConfig config) AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, UseCookies = false, // Disable cookies UseProxy = false + // ,ServerCertificateCustomValidationCallback = (msg, cert, certChain, errorsPolicy) => true }; } // special logic for .NET framework 4.7.1 that @@ -181,15 +182,15 @@ internal HttpMessageHandler SetupCustomHttpHandler(HttpClientConfig config) { // Get the original entry entry = bypassList[i].Trim(); - // . -> [.] because . means any char + // . -> [.] because . means any char entry = entry.Replace(".", "[.]"); // * -> .* because * is a quantifier and need a char or group to apply to entry = entry.Replace("*", ".*"); - + entry = entry.StartsWith("^") ? entry : $"^{entry}"; - + entry = entry.EndsWith("$") ? entry : $"{entry}$"; - + // Replace with the valid entry syntax bypassList[i] = entry; @@ -384,7 +385,7 @@ protected override async Task SendAsync(HttpRequestMessage if (httpTimeout.Ticks == 0) childCts.Cancel(); else - childCts.CancelAfter(httpTimeout); + childCts.CancelAfter(httpTimeout); } response = await base.SendAsync(requestMessage, childCts == null ? cancellationToken : childCts.Token).ConfigureAwait(false); diff --git a/Snowflake.Data/Core/RestResponse.cs b/Snowflake.Data/Core/RestResponse.cs index 75f1698ea..dbe10d858 100755 --- a/Snowflake.Data/Core/RestResponse.cs +++ b/Snowflake.Data/Core/RestResponse.cs @@ -15,10 +15,10 @@ abstract class BaseRestResponse { [JsonProperty(PropertyName = "message")] internal String message { get; set; } - + [JsonProperty(PropertyName = "code", NullValueHandling = NullValueHandling.Ignore)] internal int code { get; set; } - + [JsonProperty(PropertyName = "success")] internal bool success { get; set; } @@ -222,7 +222,7 @@ internal class QueryExecResponseData : IQueryExecResponseData // multiple statements response data [JsonProperty(PropertyName = "resultIds", NullValueHandling = NullValueHandling.Ignore)] internal string resultIds { get; set; } - + [JsonProperty(PropertyName = "queryResultFormat", NullValueHandling = NullValueHandling.Ignore)] [JsonConverter(typeof(StringEnumConverter))] internal ResultFormat queryResultFormat { get; set; } @@ -295,8 +295,44 @@ internal class ExecResponseRowType [JsonProperty(PropertyName = "nullable")] internal bool nullable { get; set; } + + [JsonProperty(PropertyName = "fields")] + internal List fields { get; set; }// = new List(); + } + + internal class FieldMetadata + { + [JsonProperty(PropertyName = "name")] + internal string name { get; set; } + + [JsonProperty(PropertyName = "byteLength", NullValueHandling = NullValueHandling.Ignore)] + private Int64 byteLength { get; set; } + + [JsonProperty(PropertyName = "typeName")] + internal string typeName { get; set; } + + [JsonProperty(PropertyName = "type")] + internal string type { get; set; } + + [JsonProperty(PropertyName = "scale", NullValueHandling = NullValueHandling.Ignore)] + internal Int64 scale { get; set; } + + [JsonProperty(PropertyName = "precision", NullValueHandling = NullValueHandling.Ignore)] + internal Int64 precision { get; set; } + + [JsonProperty(PropertyName = "nullable")] + internal bool nullable { get; set; } + + [JsonProperty(PropertyName = "fixed")] + internal bool isFixed { get; set; } + + [JsonProperty(PropertyName = "base")] + internal SFDataType baseType { get; set; } // TODO: use enum + + [JsonProperty(PropertyName = "fields")] + internal List fields { get; set; }// = new List(); } - + internal class ExecResponseChunk { [JsonProperty(PropertyName = "url")] @@ -483,4 +519,4 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s throw new NotImplementedException(); } } -} +} diff --git a/Snowflake.Data/Core/SFDataConverter.cs b/Snowflake.Data/Core/SFDataConverter.cs index 6822f03f4..b405d735f 100755 --- a/Snowflake.Data/Core/SFDataConverter.cs +++ b/Snowflake.Data/Core/SFDataConverter.cs @@ -14,7 +14,7 @@ namespace Snowflake.Data.Core public enum SFDataType { None, FIXED, REAL, TEXT, DATE, VARIANT, TIMESTAMP_LTZ, TIMESTAMP_NTZ, - TIMESTAMP_TZ, OBJECT, BINARY, TIME, BOOLEAN, ARRAY + TIMESTAMP_TZ, OBJECT, BINARY, TIME, BOOLEAN, ARRAY, MAP } static class SFDataConverter @@ -53,7 +53,7 @@ internal static object ConvertToCSharpVal(UTF8Buffer srcVal, SFDataType srcType, try { // The most common conversions are checked first for maximum performance - if (destType == typeof(Int64)) + if (destType == typeof(Int64)) { return FastParser.FastParseInt64(srcVal.Buffer, srcVal.offset, srcVal.length); } @@ -117,7 +117,7 @@ internal static object ConvertToCSharpVal(UTF8Buffer srcVal, SFDataType srcType, } else if (destType == typeof(char[])) { - byte[] data = srcType == SFDataType.BINARY ? + byte[] data = srcType == SFDataType.BINARY ? HexToBytes(srcVal.ToString()) : srcVal.GetBytes(); return Encoding.UTF8.GetString(data).ToCharArray(); } @@ -138,7 +138,7 @@ private static object ConvertToTimeSpan(UTF8Buffer srcVal, SFDataType srcType) { case SFDataType.TIME: // Convert fractional seconds since midnight to TimeSpan - // A single tick represents one hundred nanoseconds or one ten-millionth of a second. + // A single tick represents one hundred nanoseconds or one ten-millionth of a second. // There are 10,000 ticks in a millisecond return TimeSpan.FromTicks(GetTicksFromSecondAndNanosecond(srcVal)); default: @@ -183,32 +183,32 @@ private static DateTimeOffset ConvertToDateTimeOffset(UTF8Buffer srcVal, SFDataT TimeSpan offSetTimespan = new TimeSpan((offset - 1440) / 60, 0, 0); return new DateTimeOffset(UnixEpoch.Ticks + GetTicksFromSecondAndNanosecond(timeVal), TimeSpan.Zero).ToOffset(offSetTimespan); } - case SFDataType.TIMESTAMP_LTZ: + case SFDataType.TIMESTAMP_LTZ: return new DateTimeOffset(UnixEpoch.Ticks + - GetTicksFromSecondAndNanosecond(srcVal), TimeSpan.Zero).ToLocalTime(); + GetTicksFromSecondAndNanosecond(srcVal), TimeSpan.Zero).ToLocalTime(); default: - throw new SnowflakeDbException(SFError.INVALID_DATA_CONVERSION, srcVal, + throw new SnowflakeDbException(SFError.INVALID_DATA_CONVERSION, srcVal, srcType, typeof(DateTimeOffset).ToString()); } } - static int[] powersOf10 = { - 1, - 10, - 100, - 1000, - 10000, - 100000, - 1000000, - 10000000, - 100000000 + static int[] powersOf10 = { + 1, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000 }; /// /// Convert the time value with the format seconds.nanoseconds to a number of /// Ticks. A single tick represents one hundred nanoseconds. - /// For example, "23:59:59.123456789" is represented by 86399.123456789 and is + /// For example, "23:59:59.123456789" is represented by 86399.123456789 and is /// 863991234567 ticks (precision is maximum 7). /// /// The source data returned by the server. For example '86399.123456789' diff --git a/Snowflake.Data/Core/SFResultSetMetaData.cs b/Snowflake.Data/Core/SFResultSetMetaData.cs index 4a8c5651c..119a4e4a4 100755 --- a/Snowflake.Data/Core/SFResultSetMetaData.cs +++ b/Snowflake.Data/Core/SFResultSetMetaData.cs @@ -31,7 +31,7 @@ class SFResultSetMetaData internal readonly SFStatementType statementType; internal readonly List> columnTypes; - + /// /// This map is used to cache column name to column index. Index is 0-based. /// @@ -96,12 +96,12 @@ internal int GetColumnIndexByName(string targetColumnName) columnNameToIndexCache[targetColumnName] = indexCounter; return indexCounter; } - indexCounter++; + indexCounter++; } } return -1; } - + internal SFDataType GetColumnTypeByIndex(int targetIndex) { return columnTypes[targetIndex].Item1; @@ -117,6 +117,12 @@ internal long GetScaleByIndex(int targetIndex) return rowTypes[targetIndex].scale; } + internal bool IsStructuredType(int targetIndex) + { + var fields = rowTypes[targetIndex].fields; + return fields != null && fields.Count > 0; + } + private SFDataType GetSFDataType(string type) { SFDataType rslt; @@ -124,7 +130,7 @@ private SFDataType GetSFDataType(string type) return rslt; throw new SnowflakeDbException(SFError.INTERNAL_ERROR, - $"Unknow column type: {type}"); + $"Unknow column type: {type}"); } private Type GetNativeTypeForColumn(SFDataType sfType, ExecResponseRowType col) @@ -138,7 +144,7 @@ private Type GetNativeTypeForColumn(SFDataType sfType, ExecResponseRowType col) case SFDataType.TEXT: case SFDataType.VARIANT: case SFDataType.OBJECT: - case SFDataType.ARRAY: + case SFDataType.ARRAY: return typeof(string); case SFDataType.DATE: case SFDataType.TIME: @@ -156,10 +162,10 @@ private Type GetNativeTypeForColumn(SFDataType sfType, ExecResponseRowType col) $"Unknow column type: {sfType}"); } } - + internal Type GetCSharpTypeByIndex(int targetIndex) { - return columnTypes[targetIndex].Item2; + return columnTypes[targetIndex].Item2; } internal string GetColumnNameByIndex(int targetIndex) @@ -203,7 +209,7 @@ private SFStatementType FindStatementTypeById(long id) internal enum SFStatementType { [SFStatementTypeAttr(typeId = 0x0000)] - UNKNOWN, + UNKNOWN, [SFStatementTypeAttr(typeId = 0x1000)] SELECT, @@ -212,7 +218,7 @@ internal enum SFStatementType EXPLAIN, /// - /// Data Manipulation Language + /// Data Manipulation Language /// [SFStatementTypeAttr(typeId = 0x3000)] DML, @@ -257,7 +263,7 @@ internal enum SFStatementType /// Transaction Command Language /// [SFStatementTypeAttr(typeId = 0x5000)] - TCL, + TCL, /// /// Data Definition Language