Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SNOW-1271212 Fixed values uploaded to stage for bindings exceeding CLIENT_STAGE_ARRAY_BINDING_THRESHOLD #897

Merged
merged 13 commits into from
Apr 4, 2024
Merged

SNOW-1271212 Fixed values uploaded to stage for bindings exceeding CLIENT_STAGE_ARRAY_BINDING_THRESHOLD #897

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
4c539c6
SNOW-1271212 Fixed values uploaded to stage for bindings exceeding CL…
sfc-gh-mhofman Mar 26, 2024
7422b4d
SNOW-1271212 make test pass on an account unable to set CLIENT_STAGE_…
sfc-gh-mhofman Mar 26, 2024
7a91a8d
SNOW-1271212 refactoring test after review
sfc-gh-mhofman Mar 26, 2024
5c0c689
SNOW-1271212 unit test for bind uploadeer
sfc-gh-mhofman Mar 26, 2024
d3b151f
SNOW-1271212 unit tests for CSV Data Conversion
sfc-gh-mhofman Mar 27, 2024
95fd416
SNOW-1271212 fixed unit test for bind uploader conversion
sfc-gh-mhofman Mar 27, 2024
1f54e22
SNOW-1271212 added back 7 digit precision for timestamps in BindUploader
sfc-gh-mhofman Mar 28, 2024
dea9402
SNOW-1271212 simplified calculations
sfc-gh-mhofman Mar 28, 2024
22f5b42
SNOW-1271212 review fixes
sfc-gh-mhofman Mar 29, 2024
0e842c0
SNOW-1271212 refactoring of test utils
sfc-gh-mhofman Apr 2, 2024
adf3c45
SNOW-1271212 fixed epoch to be in utc
sfc-gh-mhofman Apr 3, 2024
911182f
SNOW-1271212 fixed test for lost nanos precision in conversion after …
sfc-gh-mhofman Apr 3, 2024
db80eff
SNOW-1271212 review fixes
sfc-gh-mhofman Apr 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 225 additions & 1 deletion Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
/*
* Copyright (c) 2012-2019 Snowflake Computing Inc. All rights reserved.
* Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved.
*/

using System;
using System.Data;
using System.Linq;
using Microsoft.IdentityModel.Tokens;
using Newtonsoft.Json;
using Snowflake.Data.Log;

namespace Snowflake.Data.Tests.IntegrationTests
{
Expand All @@ -14,10 +17,13 @@ namespace Snowflake.Data.Tests.IntegrationTests
using System.Text;
using System.Globalization;
using System.Collections.Generic;
using Snowflake.Data.Tests.Util;
sfc-gh-knozderko marked this conversation as resolved.
Show resolved Hide resolved

[TestFixture]
class SFBindTestIT : SFBaseTest
{
private static readonly SFLogger s_logger = SFLoggerFactory.GetLogger<SFBindTestIT>();

[Test]
public void testArrayBind()
{
Expand Down Expand Up @@ -833,5 +839,223 @@ public void testExplicitDbTypeAssignmentForArrayValue()
conn.Close();
}
}

private const string FormatYmd = "yyyy/MM/dd";
private const string FormatHms = "HH\\:mm\\:ss";
private const string FormatHmsf = "HH\\:mm\\:ss\\.fff";
private const string FormatYmdHms = "yyyy/MM/dd HH\\:mm\\:ss";
private const string FormatYmdHmsZ = "yyyy/MM/dd HH\\:mm\\:ss zzz";

// STANDARD Tables
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.DATE, null, DbType.Date, FormatYmd, null)]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIME, null, DbType.Time, FormatHms, null)]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIME, 6, DbType.Time, FormatHmsf, null)]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.DATE, null, DbType.Date, FormatYmd, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIME, null, DbType.Time, FormatHms, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIME, 6, DbType.Time, FormatHmsf, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
/* TODO: Enable when features available on the automated tests environment
// HYBRID Tables
[TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.DATE, null, DbType.Date, FormatYmd, null)]
[TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.TIME, null, DbType.Time, FormatHms, null)]
[TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.TIME, 6, DbType.Time, FormatHmsf, null)]
[TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)]
[TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
[TestCase(ResultFormat.JSON, SFTableType.Hybrid, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Hybrid, SFDataType.DATE, null, DbType.Date, FormatYmd, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Hybrid, SFDataType.TIME, null, DbType.Time, FormatHms, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Hybrid, SFDataType.TIME, 6, DbType.Time, FormatHmsf, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Hybrid, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Hybrid, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Hybrid, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
// ICEBERG Tables; require env variables: ICEBERG_EXTERNAL_VOLUME, ICEBERG_CATALOG, ICEBERG_BASE_LOCATION.
[TestCase(ResultFormat.JSON, SFTableType.Iceberg, SFDataType.DATE, null, DbType.Date, FormatYmd, null)]
[TestCase(ResultFormat.JSON, SFTableType.Iceberg, SFDataType.TIME, null, DbType.Time, FormatHms, null)]
[TestCase(ResultFormat.JSON, SFTableType.Iceberg, SFDataType.TIME, 6, DbType.Time, FormatHmsf, null)]
[TestCase(ResultFormat.JSON, SFTableType.Iceberg, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)]
// [TestCase(ResultFormat.JSON, SFTableType.Iceberg, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)] // Unsupported data type 'TIMESTAMP_TZ(6)' for iceberg tables
[TestCase(ResultFormat.JSON, SFTableType.Iceberg, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.DATE, null, DbType.Date, FormatYmd, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIME, null, DbType.Time, FormatHms, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIME, 6, DbType.Time, FormatHmsf, null)]
[TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIMESTAMP_NTZ, 6, DbType.DateTime, FormatYmdHms, null)]
// [TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIMESTAMP_TZ, 6, DbType.DateTime, FormatYmdHmsZ, null)] // Unsupported data type 'TIMESTAMP_TZ(6)' for iceberg tables
[TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
*/
// Session TimeZone cases
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")]
public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType, SFDataType columnType, Int32? columnPrecision, DbType bindingType, string comparisonFormat, string timeZone)
{
// Arrange
var timestamp = "2023/03/15 13:17:29.207 +05:00"; // 08:17:29.207 UTC
var expected = ExpectedTimestampWrapper.From(timestamp, columnType);
var columnWithPrecision = ColumnTypeWithPrecision(columnType, columnPrecision);
var testCase = $"TableType={tableType}, ColumnType={columnWithPrecision}, BindingType={bindingType}, ComparisonFormat={comparisonFormat}";
var bindingThreshold = 65280; // when exceeded enforces bindings via file on stage
var smallBatchRowCount = 2;
var bigBatchRowCount = bindingThreshold / 2;
s_logger.Info(testCase);

using (IDbConnection conn = new SnowflakeDbConnection(ConnectionString))
{
conn.Open();

conn.ExecuteNonQuery($"alter session set DOTNET_QUERY_RESULT_FORMAT = {resultFormat}");
if (!timeZone.IsNullOrEmpty()) // Driver ignores this setting and relies on local environment timezone
conn.ExecuteNonQuery($"alter session set TIMEZONE = '{timeZone}'");

CreateOrReplaceTable(conn,
TableName,
tableType.TableDDLCreationPrefix(),
new[] {
"id number(10,0) not null primary key", // necessary only for HYBRID tables
$"ts {columnWithPrecision}"
},
tableType.TableDDLCreationFlags());

// Act+Assert
var sqlInsert = $"insert into {TableName} (id, ts) values (?, ?)";
InsertSingleRecord(conn, sqlInsert, bindingType, 1, expected);
InsertMultipleRecords(conn, sqlInsert, bindingType, 2, expected, smallBatchRowCount, false);
InsertMultipleRecords(conn, sqlInsert, bindingType, smallBatchRowCount+2, expected, bigBatchRowCount, true);

// Assert
var row = 0;
using (var select = conn.CreateCommand($"select id, ts from {TableName} order by id"))
{
s_logger.Debug(select.CommandText);
var reader = select.ExecuteReader();
while (reader.Read())
{
++row;
string faultMessage = $"Mismatch for row: {row}, {testCase}";
Assert.AreEqual(row, reader.GetInt32(0));
expected.AssertEqual(reader.GetValue(1), comparisonFormat, faultMessage);
}
}
Assert.AreEqual(1+smallBatchRowCount+bigBatchRowCount, row);
}
}

private void InsertSingleRecord(IDbConnection conn, string sqlInsert, DbType binding, int identifier, ExpectedTimestampWrapper ts)
{
using (var insert = conn.CreateCommand(sqlInsert))
{
// Arrange
insert.Add("1", DbType.Int32, identifier);
if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType()))
{
var parameter = (SnowflakeDbParameter)insert.Add("2", binding, ts.GetDateTimeOffset());
parameter.SFDataType = ts.ExpectedColumnType();
}
else
{
insert.Add("2", binding, ts.GetDateTime());
}

// Act
s_logger.Info(sqlInsert);
var rowsAffected = insert.ExecuteNonQuery();

// Assert
Assert.AreEqual(1, rowsAffected);
Assert.IsNull(((SnowflakeDbCommand)insert).GetBindStage());
}
}

private void InsertMultipleRecords(IDbConnection conn, string sqlInsert, DbType binding, int initialIdentifier, ExpectedTimestampWrapper ts, int rowsCount, bool shouldUseBinding)
{
using (var insert = conn.CreateCommand(sqlInsert))
{
// Arrange
insert.Add("1", DbType.Int32, Enumerable.Range(initialIdentifier, rowsCount).ToArray());
if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType()))
{
var parameter = (SnowflakeDbParameter)insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTimeOffset(), rowsCount).ToArray());
parameter.SFDataType = ts.ExpectedColumnType();
}
else
{
insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTime(), rowsCount).ToArray());
}

// Act
s_logger.Debug(sqlInsert);
var rowsAffected = insert.ExecuteNonQuery();

// Assert
Assert.AreEqual(rowsCount, rowsAffected);
if (shouldUseBinding)
Assert.IsNotEmpty(((SnowflakeDbCommand)insert).GetBindStage());
else
Assert.IsNull(((SnowflakeDbCommand)insert).GetBindStage());
}
}

private static string ColumnTypeWithPrecision(SFDataType columnType, Int32? columnPrecision)
=> columnPrecision != null ? $"{columnType}({columnPrecision})" : $"{columnType}";
}

class ExpectedTimestampWrapper
{
private readonly SFDataType _columnType;
private readonly DateTime? _expectedDateTime;
private readonly DateTimeOffset? _expectedDateTimeOffset;

internal static ExpectedTimestampWrapper From(string timestampWithTimeZone, SFDataType columnType)
{
if (IsOffsetType(columnType))
{
var dateTimeOffset = DateTimeOffset.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture);
return new ExpectedTimestampWrapper(dateTimeOffset, columnType);
}

var dateTime = DateTime.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture);
return new ExpectedTimestampWrapper(dateTime, columnType);
}

private ExpectedTimestampWrapper(DateTime dateTime, SFDataType columnType)
{
_expectedDateTime = dateTime;
_expectedDateTimeOffset = null;
_columnType = columnType;
}

private ExpectedTimestampWrapper(DateTimeOffset dateTimeOffset, SFDataType columnType)
{
_expectedDateTimeOffset = dateTimeOffset;
_expectedDateTime = null;
_columnType = columnType;
}

internal SFDataType ExpectedColumnType() => _columnType;

internal void AssertEqual(object actual, string comparisonFormat, string faultMessage)
{
switch (_columnType)
{
case SFDataType.TIMESTAMP_TZ:
Assert.AreEqual(GetDateTimeOffset().ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage);
break;
case SFDataType.TIMESTAMP_LTZ:
Assert.AreEqual(GetDateTimeOffset().ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage);
break;
default:
Assert.AreEqual(GetDateTime().ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage);
break;
}
}

internal DateTime GetDateTime() => _expectedDateTime ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");

internal DateTimeOffset GetDateTimeOffset() => _expectedDateTimeOffset ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");

internal static bool IsOffsetType(SFDataType type) => type == SFDataType.TIMESTAMP_LTZ || type == SFDataType.TIMESTAMP_TZ;
}
}
11 changes: 10 additions & 1 deletion Snowflake.Data.Tests/SFBaseTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
using System.Runtime.InteropServices;
using NUnit.Framework;
using Snowflake.Data.Client;
using Snowflake.Data.Log;
using Snowflake.Data.Tests.Util;

[assembly:LevelOfParallelism(10)]
Expand Down Expand Up @@ -56,6 +57,8 @@ public static void TearDownContext()
#endif
public class SFBaseTestAsync
{
private static readonly SFLogger s_logger = SFLoggerFactory.GetLogger<SFBaseTestAsync>();

private const string ConnectionStringWithoutAuthFmt = "scheme={0};host={1};port={2};" +
"account={3};role={4};db={5};schema={6};warehouse={7}";
private const string ConnectionStringSnowflakeAuthFmt = ";user={0};password={1};";
Expand Down Expand Up @@ -106,10 +109,16 @@ private void RemoveTables()
}

protected void CreateOrReplaceTable(IDbConnection conn, string tableName, IEnumerable<string> columns, string additionalQueryStr = null)
{
CreateOrReplaceTable(conn, tableName, "", columns, additionalQueryStr);
}

protected void CreateOrReplaceTable(IDbConnection conn, string tableName, string tableType, IEnumerable<string> columns, string additionalQueryStr = null)
{
var columnsStr = string.Join(", ", columns);
var cmd = conn.CreateCommand();
cmd.CommandText = $"CREATE OR REPLACE TABLE {tableName}({columnsStr}) {additionalQueryStr}";
cmd.CommandText = $"CREATE OR REPLACE {tableType} TABLE {tableName}({columnsStr}) {additionalQueryStr}";
s_logger.Debug(cmd.CommandText);
cmd.ExecuteNonQuery();

_tablesToRemove.Add(tableName);
Expand Down
Loading
Loading