Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SNOW-968741: Arrow performance optimizations #831

Merged
merged 13 commits into from
Jan 3, 2024
9 changes: 4 additions & 5 deletions Snowflake.Data.Tests/IntegrationTests/SFDbDataReaderIT.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@

namespace Snowflake.Data.Tests.IntegrationTests
{
// TODO: enable tests for Arrow
//[TestFixture(ResultFormat.ARROW)]
[TestFixture(ResultFormat.ARROW)]
[TestFixture(ResultFormat.JSON)]
class SFDbDataReaderIT : SFBaseTest
{
Expand Down Expand Up @@ -571,7 +570,7 @@ public void TestGetTimestampLTZ()
}

[Test]
public void TestGetBoolean()
public void TestGetBoolean([Values]bool value)
{
using (var conn = CreateAndOpenConnection())
{
Expand All @@ -585,7 +584,7 @@ public void TestGetBoolean()
var p1 = cmd.CreateParameter();
p1.ParameterName = "1";
p1.DbType = DbType.Boolean;
p1.Value = true;
p1.Value = value;
cmd.Parameters.Add(p1);

var count = cmd.ExecuteNonQuery();
Expand All @@ -597,7 +596,7 @@ public void TestGetBoolean()
ValidateResultFormat(reader);

Assert.IsTrue(reader.Read());
Assert.IsTrue(reader.GetBoolean(0));
Assert.AreEqual(value, reader.GetBoolean(0));
reader.Close();

CloseConnection(conn);
Expand Down
33 changes: 28 additions & 5 deletions Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using Apache.Arrow;
using Apache.Arrow.Types;
Expand Down Expand Up @@ -141,7 +142,7 @@
{
var chunk = new ArrowResultChunk(_recordBatchOne);

Assert.AreEqual(0, chunk.ChunkIndex);
Assert.AreEqual(-1, chunk.ChunkIndex);
}

[Test]
Expand All @@ -156,10 +157,32 @@
[Test]
public void TestExtractCellReturnsNull()
{
var chunk = new ArrowResultChunk(RecordBatchWithNullValue);
chunk.Next();

Assert.AreEqual(DBNull.Value, chunk.ExtractCell(0, SFDataType.FIXED, 0));
var cases = new Dictionary<ArrowResultChunk, SFDataType>
{
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Int8", false, col => col.Int8(array => array.AppendNull())).Build()), SFDataType.FIXED },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Int16", false, col => col.Int16(array => array.AppendNull())).Build()), SFDataType.FIXED },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Int32", false, col => col.Int32(array => array.AppendNull())).Build()), SFDataType.FIXED },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Int64", false, col => col.Int64(array => array.AppendNull())).Build()), SFDataType.FIXED },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Decimal128", false, col => col.Decimal128(new Decimal128Type(0, 0), array => array.AppendNull())).Build()), SFDataType.FIXED },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Boolean", false, col => col.Boolean(array => array.AppendNull())).Build()), SFDataType.BOOLEAN },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Real", false, col => col.Double(array => array.AppendNull())).Build()), SFDataType.REAL },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Text", false, col => col.String(array => array.AppendNull())).Build()), SFDataType.TEXT },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Array", false, col => col.String(array => array.AppendNull())).Build()), SFDataType.ARRAY },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Variant", false, col => col.String(array => array.AppendNull())).Build()), SFDataType.VARIANT },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Object", false, col => col.String(array => array.AppendNull())).Build()), SFDataType.OBJECT },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Binary", false, col => col.Binary(array => array.AppendNull())).Build()), SFDataType.BINARY },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Date", false, col => col.Date32(array => array.AppendNull())).Build()), SFDataType.DATE },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Time", false, col => col.Int32(array => array.AppendNull())).Build()), SFDataType.TIME },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Timestamp_TZ", false, col => col.Int32(array => array.AppendNull())).Build()), SFDataType.TIMESTAMP_TZ },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Timestamp_LTZ", false, col => col.Int32(array => array.AppendNull())).Build()), SFDataType.TIMESTAMP_LTZ },
{ new ArrowResultChunk(new RecordBatch.Builder().Append("Col_Timestamp_NTZ", false, col => col.Int32(array => array.AppendNull())).Build()), SFDataType.TIMESTAMP_NTZ },
};

foreach (var (chunk, type) in cases)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, GCP)

'KeyValuePair<ArrowResultChunk, SFDataType>' does not contain a definition for 'Deconstruct' and no accessible extension method 'Deconstruct' accepting a first argument of type 'KeyValuePair<ArrowResultChunk, SFDataType>' could be found (are you missing a using directive or an assembly reference?)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, GCP)

No suitable 'Deconstruct' instance or extension method was found for type 'KeyValuePair<ArrowResultChunk, SFDataType>', with 2 out parameters and a void return type.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, GCP)

Cannot infer the type of implicitly-typed deconstruction variable 'chunk'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, GCP)

Cannot infer the type of implicitly-typed deconstruction variable 'type'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, GCP)

'KeyValuePair<ArrowResultChunk, SFDataType>' does not contain a definition for 'Deconstruct' and no accessible extension method 'Deconstruct' accepting a first argument of type 'KeyValuePair<ArrowResultChunk, SFDataType>' could be found (are you missing a using directive or an assembly reference?)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, GCP)

No suitable 'Deconstruct' instance or extension method was found for type 'KeyValuePair<ArrowResultChunk, SFDataType>', with 2 out parameters and a void return type.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, GCP)

Cannot infer the type of implicitly-typed deconstruction variable 'chunk'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, GCP)

Cannot infer the type of implicitly-typed deconstruction variable 'type'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AZURE)

'KeyValuePair<ArrowResultChunk, SFDataType>' does not contain a definition for 'Deconstruct' and no accessible extension method 'Deconstruct' accepting a first argument of type 'KeyValuePair<ArrowResultChunk, SFDataType>' could be found (are you missing a using directive or an assembly reference?)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AZURE)

No suitable 'Deconstruct' instance or extension method was found for type 'KeyValuePair<ArrowResultChunk, SFDataType>', with 2 out parameters and a void return type.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AZURE)

Cannot infer the type of implicitly-typed deconstruction variable 'chunk'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AZURE)

Cannot infer the type of implicitly-typed deconstruction variable 'type'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AZURE)

'KeyValuePair<ArrowResultChunk, SFDataType>' does not contain a definition for 'Deconstruct' and no accessible extension method 'Deconstruct' accepting a first argument of type 'KeyValuePair<ArrowResultChunk, SFDataType>' could be found (are you missing a using directive or an assembly reference?)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AZURE)

No suitable 'Deconstruct' instance or extension method was found for type 'KeyValuePair<ArrowResultChunk, SFDataType>', with 2 out parameters and a void return type.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AZURE)

Cannot infer the type of implicitly-typed deconstruction variable 'chunk'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AZURE)

Cannot infer the type of implicitly-typed deconstruction variable 'type'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AWS)

'KeyValuePair<ArrowResultChunk, SFDataType>' does not contain a definition for 'Deconstruct' and no accessible extension method 'Deconstruct' accepting a first argument of type 'KeyValuePair<ArrowResultChunk, SFDataType>' could be found (are you missing a using directive or an assembly reference?)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AWS)

No suitable 'Deconstruct' instance or extension method was found for type 'KeyValuePair<ArrowResultChunk, SFDataType>', with 2 out parameters and a void return type.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AWS)

Cannot infer the type of implicitly-typed deconstruction variable 'chunk'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AWS)

Cannot infer the type of implicitly-typed deconstruction variable 'type'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AWS)

'KeyValuePair<ArrowResultChunk, SFDataType>' does not contain a definition for 'Deconstruct' and no accessible extension method 'Deconstruct' accepting a first argument of type 'KeyValuePair<ArrowResultChunk, SFDataType>' could be found (are you missing a using directive or an assembly reference?)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AWS)

No suitable 'Deconstruct' instance or extension method was found for type 'KeyValuePair<ArrowResultChunk, SFDataType>', with 2 out parameters and a void return type.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AWS)

Cannot infer the type of implicitly-typed deconstruction variable 'chunk'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AWS)

Cannot infer the type of implicitly-typed deconstruction variable 'type'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, GCP)

'KeyValuePair<ArrowResultChunk, SFDataType>' does not contain a definition for 'Deconstruct' and no accessible extension method 'Deconstruct' accepting a first argument of type 'KeyValuePair<ArrowResultChunk, SFDataType>' could be found (are you missing a using directive or an assembly reference?)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, GCP)

No suitable 'Deconstruct' instance or extension method was found for type 'KeyValuePair<ArrowResultChunk, SFDataType>', with 2 out parameters and a void return type.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, GCP)

Cannot infer the type of implicitly-typed deconstruction variable 'chunk'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, GCP)

Cannot infer the type of implicitly-typed deconstruction variable 'type'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, GCP)

'KeyValuePair<ArrowResultChunk, SFDataType>' does not contain a definition for 'Deconstruct' and no accessible extension method 'Deconstruct' accepting a first argument of type 'KeyValuePair<ArrowResultChunk, SFDataType>' could be found (are you missing a using directive or an assembly reference?)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, GCP)

No suitable 'Deconstruct' instance or extension method was found for type 'KeyValuePair<ArrowResultChunk, SFDataType>', with 2 out parameters and a void return type.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, GCP)

Cannot infer the type of implicitly-typed deconstruction variable 'chunk'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, GCP)

Cannot infer the type of implicitly-typed deconstruction variable 'type'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AZURE)

'KeyValuePair<ArrowResultChunk, SFDataType>' does not contain a definition for 'Deconstruct' and no accessible extension method 'Deconstruct' accepting a first argument of type 'KeyValuePair<ArrowResultChunk, SFDataType>' could be found (are you missing a using directive or an assembly reference?)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AZURE)

No suitable 'Deconstruct' instance or extension method was found for type 'KeyValuePair<ArrowResultChunk, SFDataType>', with 2 out parameters and a void return type.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AZURE)

Cannot infer the type of implicitly-typed deconstruction variable 'chunk'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AZURE)

Cannot infer the type of implicitly-typed deconstruction variable 'type'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AZURE)

'KeyValuePair<ArrowResultChunk, SFDataType>' does not contain a definition for 'Deconstruct' and no accessible extension method 'Deconstruct' accepting a first argument of type 'KeyValuePair<ArrowResultChunk, SFDataType>' could be found (are you missing a using directive or an assembly reference?)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AZURE)

No suitable 'Deconstruct' instance or extension method was found for type 'KeyValuePair<ArrowResultChunk, SFDataType>', with 2 out parameters and a void return type.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AZURE)

Cannot infer the type of implicitly-typed deconstruction variable 'chunk'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AZURE)

Cannot infer the type of implicitly-typed deconstruction variable 'type'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AWS)

'KeyValuePair<ArrowResultChunk, SFDataType>' does not contain a definition for 'Deconstruct' and no accessible extension method 'Deconstruct' accepting a first argument of type 'KeyValuePair<ArrowResultChunk, SFDataType>' could be found (are you missing a using directive or an assembly reference?)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AWS)

No suitable 'Deconstruct' instance or extension method was found for type 'KeyValuePair<ArrowResultChunk, SFDataType>', with 2 out parameters and a void return type.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AWS)

Cannot infer the type of implicitly-typed deconstruction variable 'chunk'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AWS)

Cannot infer the type of implicitly-typed deconstruction variable 'type'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AWS)

'KeyValuePair<ArrowResultChunk, SFDataType>' does not contain a definition for 'Deconstruct' and no accessible extension method 'Deconstruct' accepting a first argument of type 'KeyValuePair<ArrowResultChunk, SFDataType>' could be found (are you missing a using directive or an assembly reference?)

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AWS)

No suitable 'Deconstruct' instance or extension method was found for type 'KeyValuePair<ArrowResultChunk, SFDataType>', with 2 out parameters and a void return type.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AWS)

Cannot infer the type of implicitly-typed deconstruction variable 'chunk'.

Check failure on line 181 in Snowflake.Data.Tests/UnitTests/ArrowResultChunkTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AWS)

Cannot infer the type of implicitly-typed deconstruction variable 'type'.
{
chunk.Next();
Assert.AreEqual(DBNull.Value, chunk.ExtractCell(0, type, 0), $"Expected DBNull.Value for SFDataType: {type}");
}
}

[Test]
Expand Down
Loading
Loading