From 9fe83a1da5c200d247aaa36a87582355d90a0790 Mon Sep 17 00:00:00 2001 From: Dariusz Stempniak Date: Mon, 28 Oct 2024 12:20:26 +0100 Subject: [PATCH 1/6] SNOW-1672654 fix for empty encryptionMaterial when client side encryption is disabled --- .../IntegrationTests/SFPutGetTest.cs | 92 ++++++++++--------- .../UnitTests/SFAzureClientTest.cs | 1 - .../UnitTests/SFGCSClientTest.cs | 1 - .../UnitTests/SFRemoteStorageClientTest.cs | 1 - .../UnitTests/SFS3ClientTest.cs | 1 - .../Core/FileTransfer/SFFileTransferAgent.cs | 31 ++++--- Snowflake.Data/Core/RestResponse.cs | 6 +- 7 files changed, 72 insertions(+), 61 deletions(-) diff --git a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs index 975d041e0..ac6510c9e 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs @@ -31,6 +31,7 @@ class SFPutGetTest : SFBaseTest [ThreadStatic] private static string t_schemaName; [ThreadStatic] private static string t_tableName; [ThreadStatic] private static string t_stageName; + [ThreadStatic] private static string t_stageNameSse; // server side encryption without client side encryption [ThreadStatic] private static string t_fileName; [ThreadStatic] private static string t_outputFileName; [ThreadStatic] private static string t_inputFilePath; @@ -41,12 +42,13 @@ class SFPutGetTest : SFBaseTest [ThreadStatic] private static string t_destCompressionType; [ThreadStatic] private static bool t_autoCompress; [ThreadStatic] private static List t_filesToDelete; - + public enum StageType { USER, TABLE, - NAMED + NAMED, + NAMED_SSE // without client side encryption } [OneTimeSetUp] @@ -63,7 +65,7 @@ public static void OneTimeTearDown() // Delete temp output directory and downloaded files Directory.Delete(s_outputDirectory, true); } - + [SetUp] public void SetUp() { @@ -73,6 +75,7 @@ public void SetUp() t_schemaName = testConfig.schema; t_tableName = $"TABLE_{threadSuffix}"; t_stageName = $"STAGE_{threadSuffix}"; + t_stageNameSse = $"STAGE_{threadSuffix}_SSE"; t_filesToDelete = new List(); using (var conn = new SnowflakeDbConnection(ConnectionString)) @@ -88,6 +91,10 @@ public void SetUp() // Create temp stage command.CommandText = $"CREATE OR REPLACE STAGE {t_schemaName}.{t_stageName}"; command.ExecuteNonQuery(); + + // Create temp stage without client side encryption + command.CommandText = $"CREATE OR REPLACE STAGE {t_schemaName}.{t_stageNameSse} ENCRYPTION = (TYPE = 'SNOWFLAKE_SSE')"; + command.ExecuteNonQuery(); } } } @@ -109,7 +116,7 @@ public void TearDown() command.ExecuteNonQuery(); } } - + // Delete temp files if necessary if (t_filesToDelete != null) { @@ -130,7 +137,7 @@ public void TestPutFileAsteriskWildcard() $"{absolutePathPrefix}_three.csv" }; PrepareFileData(files); - + // Set the PUT query variables t_inputFilePath = $"{absolutePathPrefix}*"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; @@ -142,7 +149,7 @@ public void TestPutFileAsteriskWildcard() VerifyFilesAreUploaded(conn, files, t_internalStagePath); } } - + [Test] public void TestPutFileAsteriskWildcardWithExtension() { @@ -167,7 +174,7 @@ public void TestPutFileAsteriskWildcardWithExtension() VerifyFilesAreUploaded(conn, files, t_internalStagePath); } } - + [Test] public void TestPutFileQuestionMarkWildcard() { @@ -180,7 +187,7 @@ public void TestPutFileQuestionMarkWildcard() PrepareFileData(files); // Create file which should be omitted during the transfer PrepareFileData($"{absolutePathPrefix}_four.csv"); - + // Set the PUT query variables t_inputFilePath = $"{absolutePathPrefix}_?.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; @@ -192,14 +199,14 @@ public void TestPutFileQuestionMarkWildcard() VerifyFilesAreUploaded(conn, files, t_internalStagePath); } } - + [Test] public void TestPutFileRelativePathWithoutDirectory() { // Set the PUT query variables t_inputFilePath = $"{Guid.NewGuid()}_1.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; - + PrepareFileData(t_inputFilePath); using (var conn = new SnowflakeDbConnection(ConnectionString)) @@ -226,7 +233,7 @@ public void TestPutGetOnClosedConnectionThrowsWithoutQueryId([Values("GET", "PUT SnowflakeDbExceptionAssert.HasErrorCode(snowflakeDbException, SFError.EXECUTE_COMMAND_ON_CLOSED_CONNECTION); } } - + [Test] public void TestGetNonExistentFileReturnsFalseAndDoesNotThrow() { @@ -236,7 +243,7 @@ public void TestGetNonExistentFileReturnsFalseAndDoesNotThrow() // Act using (var conn = new SnowflakeDbConnection(ConnectionString)) { - conn.Open(); + conn.Open(); var sql = $"GET {t_internalStagePath}/{t_fileName} file://{s_outputDirectory}"; using (var command = conn.CreateCommand()) { @@ -246,7 +253,7 @@ public void TestGetNonExistentFileReturnsFalseAndDoesNotThrow() } } } - + [Test] public void TestPutNonExistentFileThrowsWithQueryId() { @@ -256,14 +263,14 @@ public void TestPutNonExistentFileThrowsWithQueryId() // Act using (var conn = new SnowflakeDbConnection(ConnectionString)) { - conn.Open(); + conn.Open(); var snowflakeDbException = Assert.Throws(() => PutFile(conn)); Assert.IsNotNull(snowflakeDbException); Assert.IsNotNull(snowflakeDbException.QueryId); SnowflakeDbExceptionAssert.HasErrorCode(snowflakeDbException, SFError.IO_ERROR_ON_GETPUT_COMMAND); } } - + [Test] public void TestPutFileProvidesQueryIdOnFailure() { @@ -285,7 +292,7 @@ public void TestPutFileProvidesQueryIdOnFailure() SnowflakeDbExceptionAssert.HasErrorCode(snowflakeDbException, SFError.IO_ERROR_ON_GETPUT_COMMAND); } } - + [Test] public void TestPutFileWithSyntaxErrorProvidesQueryIdOnFailure() { @@ -308,7 +315,7 @@ public void TestPutFileWithSyntaxErrorProvidesQueryIdOnFailure() Assert.That(snowflakeDbException.InnerException, Is.Null); } } - + [Test] public void TestPutFileProvidesQueryIdOnSuccess() { @@ -323,7 +330,7 @@ public void TestPutFileProvidesQueryIdOnSuccess() { conn.Open(); var queryId = PutFile(conn); - + // Assert Assert.IsNotNull(queryId); Assert.DoesNotThrow(()=>Guid.Parse(queryId)); @@ -337,11 +344,11 @@ public void TestPutFileRelativePathWithDirectory() var guid = Guid.NewGuid(); var relativePath = $"{guid}"; Directory.CreateDirectory(relativePath); - + // Set the PUT query variables t_inputFilePath = $"{relativePath}{Path.DirectorySeparatorChar}{guid}_1.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; - + PrepareFileData(t_inputFilePath); using (var conn = new SnowflakeDbConnection(ConnectionString)) @@ -351,7 +358,7 @@ public void TestPutFileRelativePathWithDirectory() VerifyFilesAreUploaded(conn, new List { t_inputFilePath }, t_internalStagePath); } } - + [Test] public void TestPutFileRelativePathAsteriskWildcard() { @@ -374,7 +381,7 @@ public void TestPutFileRelativePathAsteriskWildcard() VerifyFilesAreUploaded(conn, files, t_internalStagePath); } } - + [Test] // presigned url is enabled on CI so we need to disable the test // it should be enabled when downscoped credential is the default option @@ -384,7 +391,7 @@ public void TestPutFileWithoutOverwriteFlagSkipsSecondUpload() // Set the PUT query variables t_inputFilePath = $"{Guid.NewGuid()}.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; - + PrepareFileData(t_inputFilePath); using (var conn = new SnowflakeDbConnection(ConnectionString)) @@ -395,18 +402,18 @@ public void TestPutFileWithoutOverwriteFlagSkipsSecondUpload() PutFile(conn, expectedStatus: ResultStatus.SKIPPED); } } - + [Test] public void TestPutFileWithOverwriteFlagRunsSecondUpload() { var overwriteAttribute = "OVERWRITE=TRUE"; - + // Set the PUT query variables t_inputFilePath = $"{Guid.NewGuid()}.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; - + PrepareFileData(t_inputFilePath); - + using (var conn = new SnowflakeDbConnection(ConnectionString)) { conn.Open(); @@ -415,7 +422,7 @@ public void TestPutFileWithOverwriteFlagRunsSecondUpload() PutFile(conn, overwriteAttribute, expectedStatus: ResultStatus.UPLOADED); } } - + [Test] public void TestPutDirectoryAsteriskWildcard() { @@ -431,7 +438,7 @@ public void TestPutDirectoryAsteriskWildcard() PrepareFileData(fullPath); files.Add(fullPath); } - + // Set the PUT query variables t_inputFilePath = $"{path}*{Path.DirectorySeparatorChar}*"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; @@ -459,7 +466,7 @@ public void TestPutDirectoryQuestionMarkWildcard() PrepareFileData(fullPath); files.Add(fullPath); } - + // Set the PUT query variables t_inputFilePath = $"{path}_?{Path.DirectorySeparatorChar}{guid}_?_file.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; @@ -471,7 +478,7 @@ public void TestPutDirectoryQuestionMarkWildcard() VerifyFilesAreUploaded(conn, files, t_internalStagePath); } } - + [Test] public void TestPutDirectoryMixedWildcard() { @@ -487,7 +494,7 @@ public void TestPutDirectoryMixedWildcard() PrepareFileData(fullPath); files.Add(fullPath); } - + // Set the PUT query variables t_inputFilePath = $"{path}_*{Path.DirectorySeparatorChar}{guid}_?_file.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; @@ -499,7 +506,7 @@ public void TestPutDirectoryMixedWildcard() VerifyFilesAreUploaded(conn, files, t_internalStagePath); } } - + [Test] public void TestPutGetCommand( [Values("none", "gzip", "bzip2", "brotli", "deflate", "raw_deflate", "zstd")] string sourceFileCompressionType, @@ -517,7 +524,7 @@ public void TestPutGetCommand( GetFile(conn); } } - + // Test small file upload/download with GCS_USE_DOWNSCOPED_CREDENTIAL set to true [Test] [IgnoreOnEnvIs("snowflake_cloud_env", new [] { "AWS", "AZURE" })] @@ -536,14 +543,14 @@ public void TestPutGetGcsDownscopedCredential( GetFile(conn); } } - + private void PrepareTest(string sourceFileCompressionType, StageType stageType, string stagePath, bool autoCompress) { t_stageType = stageType; t_sourceCompressionType = sourceFileCompressionType; t_autoCompress = autoCompress; // Prepare temp file name with specified file extension - t_fileName = Guid.NewGuid() + ".csv" + + t_fileName = Guid.NewGuid() + ".csv" + (t_autoCompress? SFFileCompressionTypes.LookUpByName(t_sourceCompressionType).FileExtension: ""); t_inputFilePath = Path.GetTempPath() + t_fileName; if (IsCompressedByTheDriver()) @@ -572,6 +579,9 @@ private void PrepareTest(string sourceFileCompressionType, StageType stageType, case StageType.NAMED: t_internalStagePath = $"@{t_schemaName}.{t_stageName}{stagePath}"; break; + case StageType.NAMED_SSE: + t_internalStagePath = $"@{t_schemaName}.{t_stageNameSse}{stagePath}"; + break; } } @@ -579,11 +589,11 @@ private static bool IsCompressedByTheDriver() { return t_sourceCompressionType == "none" && t_autoCompress; } - + // PUT - upload file from local directory to the stage string PutFile( - SnowflakeDbConnection conn, - String additionalAttribute = "", + SnowflakeDbConnection conn, + String additionalAttribute = "", ResultStatus expectedStatus = ResultStatus.UPLOADED) { string queryId; @@ -704,7 +714,7 @@ private void ProcessFile(String command, SnowflakeDbConnection connection) { switch (command) { - case "GET": + case "GET": GetFile(connection); break; case "PUT": @@ -747,7 +757,7 @@ private static void PrepareFileData(string file) // Prepare csv raw data and write to temp files var rawDataRow = string.Join(",", s_colData) + "\n"; var rawData = string.Concat(Enumerable.Repeat(rawDataRow, NumberOfRows)); - + File.WriteAllText(file, rawData); t_filesToDelete.Add(file); } diff --git a/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs index a1c791071..08b85a9b5 100644 --- a/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs @@ -69,7 +69,6 @@ class SFAzureClientTest : SFBaseTest stageInfo = new PutGetStageInfo() { endPoint = EndPoint, - isClientSideEncrypted = true, location = Location, locationType = SFRemoteStorageUtil.AZURE_FS, path = LocationPath, diff --git a/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs index 0fad57542..d47742743 100644 --- a/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs @@ -62,7 +62,6 @@ class SFGCSClientTest : SFBaseTest stageInfo = new PutGetStageInfo() { endPoint = null, - isClientSideEncrypted = true, location = Location, locationType = SFRemoteStorageUtil.GCS_FS, path = LocationPath, diff --git a/Snowflake.Data.Tests/UnitTests/SFRemoteStorageClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFRemoteStorageClientTest.cs index 0e9d53767..76ec7c557 100644 --- a/Snowflake.Data.Tests/UnitTests/SFRemoteStorageClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFRemoteStorageClientTest.cs @@ -87,7 +87,6 @@ class SFRemoteStorageClientTest : SFBaseTest stageInfo = new PutGetStageInfo() { endPoint = EndPoint, - isClientSideEncrypted = true, location = Location, locationType = SFRemoteStorageUtil.GCS_FS, path = LocationPath, diff --git a/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs index 54647db8b..5432b0121 100644 --- a/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs @@ -87,7 +87,6 @@ class SFS3ClientTest : SFBaseTest stageInfo = new PutGetStageInfo() { endPoint = Endpoint, - isClientSideEncrypted = true, location = Location, locationType = SFRemoteStorageUtil.S3_FS, path = LocationPath, diff --git a/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs b/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs index b27daa51f..5d56b8db5 100644 --- a/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs +++ b/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs @@ -468,7 +468,7 @@ private void updatePresignedUrl() fileMeta.stageInfo = response.data.stageInfo; fileMeta.presignedUrl = response.data.stageInfo.presignedUrl; - } + } } else if (CommandTypes.DOWNLOAD == CommandType) { @@ -477,7 +477,7 @@ private void updatePresignedUrl() FilesMetas[index].presignedUrl = TransferMetadata.presignedUrls[index]; } } - } + } } /// @@ -544,7 +544,10 @@ private void initEncryptionMaterial() { if (CommandTypes.UPLOAD == CommandType) { - EncryptionMaterials.Add(TransferMetadata.encryptionMaterial[0]); + if (TransferMetadata.isClientSideEncrypted) + { + EncryptionMaterials.Add(TransferMetadata.encryptionMaterial[0]); + } } } @@ -670,7 +673,9 @@ private void initFileMetadata( overwrite = TransferMetadata.overwrite, presignedUrl = TransferMetadata.stageInfo.presignedUrl, parallel = TransferMetadata.parallel, - encryptionMaterial = TransferMetadata.encryptionMaterial[index], + encryptionMaterial = TransferMetadata.isClientSideEncrypted + ? TransferMetadata.encryptionMaterial[index] + : null, MaxBytesInMemory = GetFileTransferMaxBytesInMemory(), _operationType = CommandTypes.DOWNLOAD }; @@ -715,7 +720,7 @@ private int GetFileTransferMaxBytesInMemory() return FileTransferConfiguration.DefaultMaxBytesInMemory; } } - + /// /// Expand the wildcards if any to generate the list of paths for all files matched by the wildcards. /// Also replace the relative paths to the absolute paths for the files if needed. @@ -731,7 +736,7 @@ private List expandFileNames(string location) var directoryName = Path.GetDirectoryName(location); var foundDirectories = ExpandDirectories(directoryName); var filePaths = new List(); - + if (ContainsWildcard(fileName)) { foreach (var directory in foundDirectories) @@ -756,8 +761,8 @@ private List expandFileNames(string location) { filePaths.AddRange( Directory.GetFiles( - directory, - fileName, + directory, + fileName, SearchOption.TopDirectoryOnly)); } } @@ -788,7 +793,7 @@ private List expandFileNames(string location) return filePaths; } - + /// /// Expand the wildcards in the directory path to generate the list of directories to be searched for the files. /// @@ -803,7 +808,7 @@ private static IEnumerable ExpandDirectories(string directoryPath) { return new List { Path.GetFullPath(directoryPath) + Path.DirectorySeparatorChar }; } - + var pathParts = directoryPath.Split(Path.DirectorySeparatorChar); var resolvedPaths = new List(); @@ -863,7 +868,7 @@ private static IEnumerable ExpandDirectories(string directoryPath) private static string ExpandHomeDirectoryIfNeeded(string directoryPath) { if (!directoryPath.Contains('~')) return directoryPath; - + var homePath = (Environment.OSVersion.Platform == PlatformID.Unix || Environment.OSVersion.Platform == PlatformID.MacOSX) ? Environment.GetEnvironmentVariable("HOME") @@ -1036,7 +1041,7 @@ private async Task UploadFilesInSequentialAsync( { await updatePresignedUrlAsync(cancellationToken).ConfigureAwait(false); } - + // Break out of loop if file is successfully uploaded or already exists if (fileMetadata.resultStatus == ResultStatus.UPLOADED.ToString() || fileMetadata.resultStatus == ResultStatus.SKIPPED.ToString()) @@ -1429,7 +1434,7 @@ private void initFileMetadataForUpload() throw new ArgumentException("No file found for: " + TransferMetadata.src_locations[0].ToString()); } } - + private static bool IsDirectory(string path) { var attr = File.GetAttributes(path); diff --git a/Snowflake.Data/Core/RestResponse.cs b/Snowflake.Data/Core/RestResponse.cs index 64275fa42..47dafa2c4 100755 --- a/Snowflake.Data/Core/RestResponse.cs +++ b/Snowflake.Data/Core/RestResponse.cs @@ -382,6 +382,9 @@ internal class PutGetResponseData : IQueryExecResponseData [JsonProperty(PropertyName = "stageInfo", NullValueHandling = NullValueHandling.Ignore)] internal PutGetStageInfo stageInfo { get; set; } + [JsonProperty(PropertyName = "isClientSideEncrypted", NullValueHandling = NullValueHandling.Ignore)] + internal bool isClientSideEncrypted { get; set; } + [JsonProperty(PropertyName = "encryptionMaterial", NullValueHandling = NullValueHandling.Ignore)] [JsonConverter(typeof(SingleOrArrayConverter))] internal List encryptionMaterial { get; set; } @@ -428,9 +431,6 @@ internal class PutGetStageInfo [JsonProperty(PropertyName = "storageAccount", NullValueHandling = NullValueHandling.Ignore)] internal string storageAccount { get; set; } - [JsonProperty(PropertyName = "isClientSideEncrypted", NullValueHandling = NullValueHandling.Ignore)] - internal bool isClientSideEncrypted { get; set; } - [JsonProperty(PropertyName = "creds", NullValueHandling = NullValueHandling.Ignore)] internal Dictionary stageCredentials { get; set; } From 10f30100582e6509833ce98a0a285f18c59667b9 Mon Sep 17 00:00:00 2001 From: Dariusz Stempniak Date: Mon, 28 Oct 2024 13:07:36 +0100 Subject: [PATCH 2/6] SNOW-1672654 fix for GCP and Amazon --- Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs | 4 ++-- Snowflake.Data/Core/RestResponse.cs | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs b/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs index 5d56b8db5..304f21376 100644 --- a/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs +++ b/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs @@ -544,7 +544,7 @@ private void initEncryptionMaterial() { if (CommandTypes.UPLOAD == CommandType) { - if (TransferMetadata.isClientSideEncrypted) + if (TransferMetadata.encryptionMaterial.Count > 0) { EncryptionMaterials.Add(TransferMetadata.encryptionMaterial[0]); } @@ -673,7 +673,7 @@ private void initFileMetadata( overwrite = TransferMetadata.overwrite, presignedUrl = TransferMetadata.stageInfo.presignedUrl, parallel = TransferMetadata.parallel, - encryptionMaterial = TransferMetadata.isClientSideEncrypted + encryptionMaterial = index < TransferMetadata.encryptionMaterial.Count ? TransferMetadata.encryptionMaterial[index] : null, MaxBytesInMemory = GetFileTransferMaxBytesInMemory(), diff --git a/Snowflake.Data/Core/RestResponse.cs b/Snowflake.Data/Core/RestResponse.cs index 47dafa2c4..47088e9d0 100755 --- a/Snowflake.Data/Core/RestResponse.cs +++ b/Snowflake.Data/Core/RestResponse.cs @@ -382,9 +382,6 @@ internal class PutGetResponseData : IQueryExecResponseData [JsonProperty(PropertyName = "stageInfo", NullValueHandling = NullValueHandling.Ignore)] internal PutGetStageInfo stageInfo { get; set; } - [JsonProperty(PropertyName = "isClientSideEncrypted", NullValueHandling = NullValueHandling.Ignore)] - internal bool isClientSideEncrypted { get; set; } - [JsonProperty(PropertyName = "encryptionMaterial", NullValueHandling = NullValueHandling.Ignore)] [JsonConverter(typeof(SingleOrArrayConverter))] internal List encryptionMaterial { get; set; } From a738b8e9c13f26b643c1c30615e9451dd0572989 Mon Sep 17 00:00:00 2001 From: Dariusz Stempniak Date: Mon, 28 Oct 2024 14:05:34 +0100 Subject: [PATCH 3/6] SNOW-1672654 fix 2 --- .../IntegrationTests/SFPutGetTest.cs | 21 ++++++++++++++++++- .../Core/FileTransfer/SFFileTransferAgent.cs | 2 +- Snowflake.Data/Core/RestResponse.cs | 3 +++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs index ac6510c9e..64df3fe96 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs @@ -510,7 +510,26 @@ public void TestPutDirectoryMixedWildcard() [Test] public void TestPutGetCommand( [Values("none", "gzip", "bzip2", "brotli", "deflate", "raw_deflate", "zstd")] string sourceFileCompressionType, - [Values] StageType stageType, + [Values(StageType.USER, StageType.TABLE, StageType.NAMED)] StageType stageType, + [Values("", "/TEST_PATH", "/DEEP/TEST_PATH")] string stagePath, + [Values] bool autoCompress) + { + PrepareTest(sourceFileCompressionType, stageType, stagePath, autoCompress); + + using (var conn = new SnowflakeDbConnection(ConnectionString)) + { + conn.Open(); + PutFile(conn); + CopyIntoTable(conn); + GetFile(conn); + } + } + + [Test] + [IgnoreOnEnvIs("snowflake_cloud_env", new [] { "GCP", "AZURE" })] + public void TestPutGetCommandWithoutClientSideEncryption( + [Values("none", "gzip", "bzip2", "brotli", "deflate", "raw_deflate", "zstd")] string sourceFileCompressionType, + [Values(StageType.NAMED_SSE)] StageType stageType, [Values("", "/TEST_PATH", "/DEEP/TEST_PATH")] string stagePath, [Values] bool autoCompress) { diff --git a/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs b/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs index 304f21376..1ee8557b6 100644 --- a/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs +++ b/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs @@ -544,7 +544,7 @@ private void initEncryptionMaterial() { if (CommandTypes.UPLOAD == CommandType) { - if (TransferMetadata.encryptionMaterial.Count > 0) + if (TransferMetadata.stageInfo.isClientSideEncrypted) { EncryptionMaterials.Add(TransferMetadata.encryptionMaterial[0]); } diff --git a/Snowflake.Data/Core/RestResponse.cs b/Snowflake.Data/Core/RestResponse.cs index 47088e9d0..64275fa42 100755 --- a/Snowflake.Data/Core/RestResponse.cs +++ b/Snowflake.Data/Core/RestResponse.cs @@ -428,6 +428,9 @@ internal class PutGetStageInfo [JsonProperty(PropertyName = "storageAccount", NullValueHandling = NullValueHandling.Ignore)] internal string storageAccount { get; set; } + [JsonProperty(PropertyName = "isClientSideEncrypted", NullValueHandling = NullValueHandling.Ignore)] + internal bool isClientSideEncrypted { get; set; } + [JsonProperty(PropertyName = "creds", NullValueHandling = NullValueHandling.Ignore)] internal Dictionary stageCredentials { get; set; } From b2e816fa823c6a4d475012cddd1e54f8d1619837 Mon Sep 17 00:00:00 2001 From: Dariusz Stempniak Date: Mon, 28 Oct 2024 14:31:14 +0100 Subject: [PATCH 4/6] SNOW-1672654 fix 3 --- .../IntegrationTests/SFPutGetTest.cs | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs index 64df3fe96..60503f281 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs @@ -47,8 +47,7 @@ public enum StageType { USER, TABLE, - NAMED, - NAMED_SSE // without client side encryption + NAMED } [OneTimeSetUp] @@ -510,7 +509,7 @@ public void TestPutDirectoryMixedWildcard() [Test] public void TestPutGetCommand( [Values("none", "gzip", "bzip2", "brotli", "deflate", "raw_deflate", "zstd")] string sourceFileCompressionType, - [Values(StageType.USER, StageType.TABLE, StageType.NAMED)] StageType stageType, + [Values] StageType stageType, [Values("", "/TEST_PATH", "/DEEP/TEST_PATH")] string stagePath, [Values] bool autoCompress) { @@ -527,13 +526,12 @@ public void TestPutGetCommand( [Test] [IgnoreOnEnvIs("snowflake_cloud_env", new [] { "GCP", "AZURE" })] - public void TestPutGetCommandWithoutClientSideEncryption( + public void TestPutGetCommandForNamedStageWithoutClientSideEncryption( [Values("none", "gzip", "bzip2", "brotli", "deflate", "raw_deflate", "zstd")] string sourceFileCompressionType, - [Values(StageType.NAMED_SSE)] StageType stageType, [Values("", "/TEST_PATH", "/DEEP/TEST_PATH")] string stagePath, [Values] bool autoCompress) { - PrepareTest(sourceFileCompressionType, stageType, stagePath, autoCompress); + PrepareTest(sourceFileCompressionType, StageType.NAMED, stagePath, autoCompress, false); using (var conn = new SnowflakeDbConnection(ConnectionString)) { @@ -563,7 +561,8 @@ public void TestPutGetGcsDownscopedCredential( } } - private void PrepareTest(string sourceFileCompressionType, StageType stageType, string stagePath, bool autoCompress) + private void PrepareTest(string sourceFileCompressionType, StageType stageType, string stagePath, + bool autoCompress, bool clientEncryption = true) { t_stageType = stageType; t_sourceCompressionType = sourceFileCompressionType; @@ -596,10 +595,9 @@ private void PrepareTest(string sourceFileCompressionType, StageType stageType, t_internalStagePath = $"@{t_schemaName}.%{t_tableName}{stagePath}"; break; case StageType.NAMED: - t_internalStagePath = $"@{t_schemaName}.{t_stageName}{stagePath}"; - break; - case StageType.NAMED_SSE: - t_internalStagePath = $"@{t_schemaName}.{t_stageNameSse}{stagePath}"; + t_internalStagePath = clientEncryption + ? $"@{t_schemaName}.{t_stageName}{stagePath}" + : $"@{t_schemaName}.{t_stageNameSse}{stagePath}"; break; } } From 597e40306ad25e6cf36a2865e7f1d9536727341c Mon Sep 17 00:00:00 2001 From: Dariusz Stempniak Date: Mon, 28 Oct 2024 15:03:19 +0100 Subject: [PATCH 5/6] SNOW-1672654 fix 4 --- Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs index 60503f281..a6af5b6e1 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs @@ -527,8 +527,8 @@ public void TestPutGetCommand( [Test] [IgnoreOnEnvIs("snowflake_cloud_env", new [] { "GCP", "AZURE" })] public void TestPutGetCommandForNamedStageWithoutClientSideEncryption( - [Values("none", "gzip", "bzip2", "brotli", "deflate", "raw_deflate", "zstd")] string sourceFileCompressionType, - [Values("", "/TEST_PATH", "/DEEP/TEST_PATH")] string stagePath, + [Values("none", "gzip")] string sourceFileCompressionType, + [Values("", "/DEEP/TEST_PATH")] string stagePath, [Values] bool autoCompress) { PrepareTest(sourceFileCompressionType, StageType.NAMED, stagePath, autoCompress, false); From 83a522735529c94c8ce42324ba607a905896c872 Mon Sep 17 00:00:00 2001 From: Dariusz Stempniak Date: Mon, 28 Oct 2024 18:43:20 +0100 Subject: [PATCH 6/6] SNOW-1672654 support for SNOWFLAKE_SSE stages in GCP and Azure --- .../IntegrationTests/SFPutGetTest.cs | 1 - .../FileTransfer/StorageClient/SFGCSClient.cs | 48 +++++++------- .../FileTransfer/StorageClient/SFS3Client.cs | 11 ++-- .../StorageClient/SFSnowflakeAzureClient.cs | 64 +++++++++++-------- 4 files changed, 69 insertions(+), 55 deletions(-) diff --git a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs index a6af5b6e1..2ef0c7ef9 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs @@ -525,7 +525,6 @@ public void TestPutGetCommand( } [Test] - [IgnoreOnEnvIs("snowflake_cloud_env", new [] { "GCP", "AZURE" })] public void TestPutGetCommandForNamedStageWithoutClientSideEncryption( [Values("none", "gzip")] string sourceFileCompressionType, [Values("", "/DEEP/TEST_PATH")] string stagePath, diff --git a/Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs b/Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs index 9e588e921..f56baf2fa 100644 --- a/Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs +++ b/Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs @@ -240,11 +240,9 @@ internal string generateFileURL(string stageLocation, string fileName) /// The encryption metadata for the header. public void UploadFile(SFFileMetadata fileMetadata, Stream fileBytesStream, SFEncryptionMetadata encryptionMetadata) { - String encryptionData = GetUploadEncryptionData(encryptionMetadata); - try { - WebRequest request = GetUploadFileRequest(fileMetadata, encryptionMetadata, encryptionData); + WebRequest request = GetUploadFileRequest(fileMetadata, encryptionMetadata); Stream dataStream = request.GetRequestStream(); fileBytesStream.Position = 0; @@ -271,11 +269,9 @@ public void UploadFile(SFFileMetadata fileMetadata, Stream fileBytesStream, SFEn /// The encryption metadata for the header. public async Task UploadFileAsync(SFFileMetadata fileMetadata, Stream fileByteStream, SFEncryptionMetadata encryptionMetadata, CancellationToken cancellationToken) { - String encryptionData = GetUploadEncryptionData(encryptionMetadata); - try { - WebRequest request = GetUploadFileRequest(fileMetadata, encryptionMetadata, encryptionData); + WebRequest request = GetUploadFileRequest(fileMetadata, encryptionMetadata); Stream dataStream = await request.GetRequestStreamAsync().ConfigureAwait(false); fileByteStream.Position = 0; @@ -294,14 +290,19 @@ public async Task UploadFileAsync(SFFileMetadata fileMetadata, Stream fileByteSt } } - private WebRequest GetUploadFileRequest(SFFileMetadata fileMetadata, SFEncryptionMetadata encryptionMetadata, String encryptionData) + private WebRequest GetUploadFileRequest(SFFileMetadata fileMetadata, SFEncryptionMetadata encryptionMetadata) { // Issue the POST/PUT request WebRequest request = _customWebRequest == null ? FormBaseRequest(fileMetadata, "PUT") : _customWebRequest; request.Headers.Add(GCS_METADATA_SFC_DIGEST, fileMetadata.sha256Digest); - request.Headers.Add(GCS_METADATA_MATDESC_KEY, encryptionMetadata.matDesc); - request.Headers.Add(GCS_METADATA_ENCRYPTIONDATAPROP, encryptionData); + if (fileMetadata.stageInfo.isClientSideEncrypted) + { + String encryptionData = GetUploadEncryptionData(ref fileMetadata, encryptionMetadata); + + request.Headers.Add(GCS_METADATA_MATDESC_KEY, encryptionMetadata.matDesc); + request.Headers.Add(GCS_METADATA_ENCRYPTIONDATAPROP, encryptionData); + } return request; } @@ -311,7 +312,7 @@ private WebRequest GetUploadFileRequest(SFFileMetadata fileMetadata, SFEncryptio /// /// The encryption metadata for the header. /// Stream content. - private String GetUploadEncryptionData(SFEncryptionMetadata encryptionMetadata) + private String GetUploadEncryptionData(ref SFFileMetadata fileMetadata, SFEncryptionMetadata encryptionMetadata) { // Create the encryption header value string encryptionData = JsonConvert.SerializeObject(new EncryptionData @@ -415,20 +416,23 @@ private void HandleDownloadResponse(HttpWebResponse response, SFFileMetadata fil WebHeaderCollection headers = response.Headers; // Get header values - dynamic encryptionData = JsonConvert.DeserializeObject(headers.Get(GCS_METADATA_ENCRYPTIONDATAPROP)); - string matDesc = headers.Get(GCS_METADATA_MATDESC_KEY); - - // Get encryption metadata from encryption data header value - SFEncryptionMetadata encryptionMetadata = null; - if (encryptionData != null) + var encryptionDataStr = headers.Get(GCS_METADATA_ENCRYPTIONDATAPROP); + if (encryptionDataStr != null) { - encryptionMetadata = new SFEncryptionMetadata + dynamic encryptionData = JsonConvert.DeserializeObject(encryptionDataStr); + string matDesc = headers.Get(GCS_METADATA_MATDESC_KEY); + + // Get encryption metadata from encryption data header value + if (encryptionData != null) { - iv = encryptionData["ContentEncryptionIV"], - key = encryptionData["WrappedContentKey"]["EncryptedKey"], - matDesc = matDesc - }; - fileMetadata.encryptionMetadata = encryptionMetadata; + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata + { + iv = encryptionData["ContentEncryptionIV"], + key = encryptionData["WrappedContentKey"]["EncryptedKey"], + matDesc = matDesc + }; + fileMetadata.encryptionMetadata = encryptionMetadata; + } } fileMetadata.sha256Digest = headers.Get(GCS_METADATA_SFC_DIGEST); diff --git a/Snowflake.Data/Core/FileTransfer/StorageClient/SFS3Client.cs b/Snowflake.Data/Core/FileTransfer/StorageClient/SFS3Client.cs index b6896cc79..ea0eb3fd0 100644 --- a/Snowflake.Data/Core/FileTransfer/StorageClient/SFS3Client.cs +++ b/Snowflake.Data/Core/FileTransfer/StorageClient/SFS3Client.cs @@ -422,10 +422,13 @@ private PutObjectRequest GetPutObjectRequest(ref AmazonS3Client client, SFFileMe ContentType = HTTP_HEADER_VALUE_OCTET_STREAM }; - // Populate the S3 Request Metadata - putObjectRequest.Metadata.Add(AMZ_META_PREFIX + AMZ_IV, encryptionMetadata.iv); - putObjectRequest.Metadata.Add(AMZ_META_PREFIX + AMZ_KEY, encryptionMetadata.key); - putObjectRequest.Metadata.Add(AMZ_META_PREFIX + AMZ_MATDESC, encryptionMetadata.matDesc); + if (stageInfo.isClientSideEncrypted) + { + // Populate the S3 Request Metadata + putObjectRequest.Metadata.Add(AMZ_META_PREFIX + AMZ_IV, encryptionMetadata.iv); + putObjectRequest.Metadata.Add(AMZ_META_PREFIX + AMZ_KEY, encryptionMetadata.key); + putObjectRequest.Metadata.Add(AMZ_META_PREFIX + AMZ_MATDESC, encryptionMetadata.matDesc); + } return putObjectRequest; } diff --git a/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs b/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs index f0ad3f09e..98c2694cb 100644 --- a/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs +++ b/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs @@ -158,13 +158,17 @@ private FileHeader HandleFileHeaderResponse(ref SFFileMetadata fileMetadata, Blo { fileMetadata.resultStatus = ResultStatus.UPLOADED.ToString(); - dynamic encryptionData = JsonConvert.DeserializeObject(response.Metadata["encryptiondata"]); - SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata + SFEncryptionMetadata encryptionMetadata = null; + if (response.Metadata.TryGetValue("encryptiondata", out var encryptionDataStr)) { - iv = encryptionData["ContentEncryptionIV"], - key = encryptionData.WrappedContentKey["EncryptedKey"], - matDesc = response.Metadata["matdesc"] - }; + dynamic encryptionData = JsonConvert.DeserializeObject(encryptionDataStr); + encryptionMetadata = new SFEncryptionMetadata + { + iv = encryptionData["ContentEncryptionIV"], + key = encryptionData.WrappedContentKey["EncryptedKey"], + matDesc = response.Metadata["matdesc"] + }; + } return new FileHeader { @@ -242,31 +246,35 @@ public async Task UploadFileAsync(SFFileMetadata fileMetadata, Stream fileBytesS /// The encryption metadata for the header. private BlobClient GetUploadFileBlobClient(ref IDictionarymetadata, SFFileMetadata fileMetadata, SFEncryptionMetadata encryptionMetadata) { - // Create the JSON for the encryption data header - string encryptionData = JsonConvert.SerializeObject(new EncryptionData + if (fileMetadata.stageInfo.isClientSideEncrypted) { - EncryptionMode = "FullBlob", - WrappedContentKey = new WrappedContentInfo - { - KeyId = "symmKey1", - EncryptedKey = encryptionMetadata.key, - Algorithm = "AES_CBC_256" - }, - EncryptionAgent = new EncryptionAgentInfo + // Create the JSON for the encryption data header + string encryptionData = JsonConvert.SerializeObject(new EncryptionData { - Protocol = "1.0", - EncryptionAlgorithm = "AES_CBC_256" - }, - ContentEncryptionIV = encryptionMetadata.iv, - KeyWrappingMetadata = new KeyWrappingMetadataInfo - { - EncryptionLibrary = "Java 5.3.0" - } - }); + EncryptionMode = "FullBlob", + WrappedContentKey = new WrappedContentInfo + { + KeyId = "symmKey1", + EncryptedKey = encryptionMetadata.key, + Algorithm = "AES_CBC_256" + }, + EncryptionAgent = new EncryptionAgentInfo + { + Protocol = "1.0", + EncryptionAlgorithm = "AES_CBC_256" + }, + ContentEncryptionIV = encryptionMetadata.iv, + KeyWrappingMetadata = new KeyWrappingMetadataInfo + { + EncryptionLibrary = "Java 5.3.0" + } + }); + + // Create the metadata to use for the header + metadata.Add("encryptiondata", encryptionData); + metadata.Add("matdesc", encryptionMetadata.matDesc); + } - // Create the metadata to use for the header - metadata.Add("encryptiondata", encryptionData); - metadata.Add("matdesc", encryptionMetadata.matDesc); metadata.Add("sfcdigest", fileMetadata.sha256Digest); PutGetStageInfo stageInfo = fileMetadata.stageInfo;