From 9fe83a1da5c200d247aaa36a87582355d90a0790 Mon Sep 17 00:00:00 2001 From: Dariusz Stempniak Date: Mon, 28 Oct 2024 12:20:26 +0100 Subject: [PATCH] SNOW-1672654 fix for empty encryptionMaterial when client side encryption is disabled --- .../IntegrationTests/SFPutGetTest.cs | 92 ++++++++++--------- .../UnitTests/SFAzureClientTest.cs | 1 - .../UnitTests/SFGCSClientTest.cs | 1 - .../UnitTests/SFRemoteStorageClientTest.cs | 1 - .../UnitTests/SFS3ClientTest.cs | 1 - .../Core/FileTransfer/SFFileTransferAgent.cs | 31 ++++--- Snowflake.Data/Core/RestResponse.cs | 6 +- 7 files changed, 72 insertions(+), 61 deletions(-) diff --git a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs index 975d041e0..ac6510c9e 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs @@ -31,6 +31,7 @@ class SFPutGetTest : SFBaseTest [ThreadStatic] private static string t_schemaName; [ThreadStatic] private static string t_tableName; [ThreadStatic] private static string t_stageName; + [ThreadStatic] private static string t_stageNameSse; // server side encryption without client side encryption [ThreadStatic] private static string t_fileName; [ThreadStatic] private static string t_outputFileName; [ThreadStatic] private static string t_inputFilePath; @@ -41,12 +42,13 @@ class SFPutGetTest : SFBaseTest [ThreadStatic] private static string t_destCompressionType; [ThreadStatic] private static bool t_autoCompress; [ThreadStatic] private static List t_filesToDelete; - + public enum StageType { USER, TABLE, - NAMED + NAMED, + NAMED_SSE // without client side encryption } [OneTimeSetUp] @@ -63,7 +65,7 @@ public static void OneTimeTearDown() // Delete temp output directory and downloaded files Directory.Delete(s_outputDirectory, true); } - + [SetUp] public void SetUp() { @@ -73,6 +75,7 @@ public void SetUp() t_schemaName = testConfig.schema; t_tableName = $"TABLE_{threadSuffix}"; t_stageName = $"STAGE_{threadSuffix}"; + t_stageNameSse = $"STAGE_{threadSuffix}_SSE"; t_filesToDelete = new List(); using (var conn = new SnowflakeDbConnection(ConnectionString)) @@ -88,6 +91,10 @@ public void SetUp() // Create temp stage command.CommandText = $"CREATE OR REPLACE STAGE {t_schemaName}.{t_stageName}"; command.ExecuteNonQuery(); + + // Create temp stage without client side encryption + command.CommandText = $"CREATE OR REPLACE STAGE {t_schemaName}.{t_stageNameSse} ENCRYPTION = (TYPE = 'SNOWFLAKE_SSE')"; + command.ExecuteNonQuery(); } } } @@ -109,7 +116,7 @@ public void TearDown() command.ExecuteNonQuery(); } } - + // Delete temp files if necessary if (t_filesToDelete != null) { @@ -130,7 +137,7 @@ public void TestPutFileAsteriskWildcard() $"{absolutePathPrefix}_three.csv" }; PrepareFileData(files); - + // Set the PUT query variables t_inputFilePath = $"{absolutePathPrefix}*"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; @@ -142,7 +149,7 @@ public void TestPutFileAsteriskWildcard() VerifyFilesAreUploaded(conn, files, t_internalStagePath); } } - + [Test] public void TestPutFileAsteriskWildcardWithExtension() { @@ -167,7 +174,7 @@ public void TestPutFileAsteriskWildcardWithExtension() VerifyFilesAreUploaded(conn, files, t_internalStagePath); } } - + [Test] public void TestPutFileQuestionMarkWildcard() { @@ -180,7 +187,7 @@ public void TestPutFileQuestionMarkWildcard() PrepareFileData(files); // Create file which should be omitted during the transfer PrepareFileData($"{absolutePathPrefix}_four.csv"); - + // Set the PUT query variables t_inputFilePath = $"{absolutePathPrefix}_?.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; @@ -192,14 +199,14 @@ public void TestPutFileQuestionMarkWildcard() VerifyFilesAreUploaded(conn, files, t_internalStagePath); } } - + [Test] public void TestPutFileRelativePathWithoutDirectory() { // Set the PUT query variables t_inputFilePath = $"{Guid.NewGuid()}_1.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; - + PrepareFileData(t_inputFilePath); using (var conn = new SnowflakeDbConnection(ConnectionString)) @@ -226,7 +233,7 @@ public void TestPutGetOnClosedConnectionThrowsWithoutQueryId([Values("GET", "PUT SnowflakeDbExceptionAssert.HasErrorCode(snowflakeDbException, SFError.EXECUTE_COMMAND_ON_CLOSED_CONNECTION); } } - + [Test] public void TestGetNonExistentFileReturnsFalseAndDoesNotThrow() { @@ -236,7 +243,7 @@ public void TestGetNonExistentFileReturnsFalseAndDoesNotThrow() // Act using (var conn = new SnowflakeDbConnection(ConnectionString)) { - conn.Open(); + conn.Open(); var sql = $"GET {t_internalStagePath}/{t_fileName} file://{s_outputDirectory}"; using (var command = conn.CreateCommand()) { @@ -246,7 +253,7 @@ public void TestGetNonExistentFileReturnsFalseAndDoesNotThrow() } } } - + [Test] public void TestPutNonExistentFileThrowsWithQueryId() { @@ -256,14 +263,14 @@ public void TestPutNonExistentFileThrowsWithQueryId() // Act using (var conn = new SnowflakeDbConnection(ConnectionString)) { - conn.Open(); + conn.Open(); var snowflakeDbException = Assert.Throws(() => PutFile(conn)); Assert.IsNotNull(snowflakeDbException); Assert.IsNotNull(snowflakeDbException.QueryId); SnowflakeDbExceptionAssert.HasErrorCode(snowflakeDbException, SFError.IO_ERROR_ON_GETPUT_COMMAND); } } - + [Test] public void TestPutFileProvidesQueryIdOnFailure() { @@ -285,7 +292,7 @@ public void TestPutFileProvidesQueryIdOnFailure() SnowflakeDbExceptionAssert.HasErrorCode(snowflakeDbException, SFError.IO_ERROR_ON_GETPUT_COMMAND); } } - + [Test] public void TestPutFileWithSyntaxErrorProvidesQueryIdOnFailure() { @@ -308,7 +315,7 @@ public void TestPutFileWithSyntaxErrorProvidesQueryIdOnFailure() Assert.That(snowflakeDbException.InnerException, Is.Null); } } - + [Test] public void TestPutFileProvidesQueryIdOnSuccess() { @@ -323,7 +330,7 @@ public void TestPutFileProvidesQueryIdOnSuccess() { conn.Open(); var queryId = PutFile(conn); - + // Assert Assert.IsNotNull(queryId); Assert.DoesNotThrow(()=>Guid.Parse(queryId)); @@ -337,11 +344,11 @@ public void TestPutFileRelativePathWithDirectory() var guid = Guid.NewGuid(); var relativePath = $"{guid}"; Directory.CreateDirectory(relativePath); - + // Set the PUT query variables t_inputFilePath = $"{relativePath}{Path.DirectorySeparatorChar}{guid}_1.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; - + PrepareFileData(t_inputFilePath); using (var conn = new SnowflakeDbConnection(ConnectionString)) @@ -351,7 +358,7 @@ public void TestPutFileRelativePathWithDirectory() VerifyFilesAreUploaded(conn, new List { t_inputFilePath }, t_internalStagePath); } } - + [Test] public void TestPutFileRelativePathAsteriskWildcard() { @@ -374,7 +381,7 @@ public void TestPutFileRelativePathAsteriskWildcard() VerifyFilesAreUploaded(conn, files, t_internalStagePath); } } - + [Test] // presigned url is enabled on CI so we need to disable the test // it should be enabled when downscoped credential is the default option @@ -384,7 +391,7 @@ public void TestPutFileWithoutOverwriteFlagSkipsSecondUpload() // Set the PUT query variables t_inputFilePath = $"{Guid.NewGuid()}.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; - + PrepareFileData(t_inputFilePath); using (var conn = new SnowflakeDbConnection(ConnectionString)) @@ -395,18 +402,18 @@ public void TestPutFileWithoutOverwriteFlagSkipsSecondUpload() PutFile(conn, expectedStatus: ResultStatus.SKIPPED); } } - + [Test] public void TestPutFileWithOverwriteFlagRunsSecondUpload() { var overwriteAttribute = "OVERWRITE=TRUE"; - + // Set the PUT query variables t_inputFilePath = $"{Guid.NewGuid()}.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; - + PrepareFileData(t_inputFilePath); - + using (var conn = new SnowflakeDbConnection(ConnectionString)) { conn.Open(); @@ -415,7 +422,7 @@ public void TestPutFileWithOverwriteFlagRunsSecondUpload() PutFile(conn, overwriteAttribute, expectedStatus: ResultStatus.UPLOADED); } } - + [Test] public void TestPutDirectoryAsteriskWildcard() { @@ -431,7 +438,7 @@ public void TestPutDirectoryAsteriskWildcard() PrepareFileData(fullPath); files.Add(fullPath); } - + // Set the PUT query variables t_inputFilePath = $"{path}*{Path.DirectorySeparatorChar}*"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; @@ -459,7 +466,7 @@ public void TestPutDirectoryQuestionMarkWildcard() PrepareFileData(fullPath); files.Add(fullPath); } - + // Set the PUT query variables t_inputFilePath = $"{path}_?{Path.DirectorySeparatorChar}{guid}_?_file.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; @@ -471,7 +478,7 @@ public void TestPutDirectoryQuestionMarkWildcard() VerifyFilesAreUploaded(conn, files, t_internalStagePath); } } - + [Test] public void TestPutDirectoryMixedWildcard() { @@ -487,7 +494,7 @@ public void TestPutDirectoryMixedWildcard() PrepareFileData(fullPath); files.Add(fullPath); } - + // Set the PUT query variables t_inputFilePath = $"{path}_*{Path.DirectorySeparatorChar}{guid}_?_file.csv"; t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; @@ -499,7 +506,7 @@ public void TestPutDirectoryMixedWildcard() VerifyFilesAreUploaded(conn, files, t_internalStagePath); } } - + [Test] public void TestPutGetCommand( [Values("none", "gzip", "bzip2", "brotli", "deflate", "raw_deflate", "zstd")] string sourceFileCompressionType, @@ -517,7 +524,7 @@ public void TestPutGetCommand( GetFile(conn); } } - + // Test small file upload/download with GCS_USE_DOWNSCOPED_CREDENTIAL set to true [Test] [IgnoreOnEnvIs("snowflake_cloud_env", new [] { "AWS", "AZURE" })] @@ -536,14 +543,14 @@ public void TestPutGetGcsDownscopedCredential( GetFile(conn); } } - + private void PrepareTest(string sourceFileCompressionType, StageType stageType, string stagePath, bool autoCompress) { t_stageType = stageType; t_sourceCompressionType = sourceFileCompressionType; t_autoCompress = autoCompress; // Prepare temp file name with specified file extension - t_fileName = Guid.NewGuid() + ".csv" + + t_fileName = Guid.NewGuid() + ".csv" + (t_autoCompress? SFFileCompressionTypes.LookUpByName(t_sourceCompressionType).FileExtension: ""); t_inputFilePath = Path.GetTempPath() + t_fileName; if (IsCompressedByTheDriver()) @@ -572,6 +579,9 @@ private void PrepareTest(string sourceFileCompressionType, StageType stageType, case StageType.NAMED: t_internalStagePath = $"@{t_schemaName}.{t_stageName}{stagePath}"; break; + case StageType.NAMED_SSE: + t_internalStagePath = $"@{t_schemaName}.{t_stageNameSse}{stagePath}"; + break; } } @@ -579,11 +589,11 @@ private static bool IsCompressedByTheDriver() { return t_sourceCompressionType == "none" && t_autoCompress; } - + // PUT - upload file from local directory to the stage string PutFile( - SnowflakeDbConnection conn, - String additionalAttribute = "", + SnowflakeDbConnection conn, + String additionalAttribute = "", ResultStatus expectedStatus = ResultStatus.UPLOADED) { string queryId; @@ -704,7 +714,7 @@ private void ProcessFile(String command, SnowflakeDbConnection connection) { switch (command) { - case "GET": + case "GET": GetFile(connection); break; case "PUT": @@ -747,7 +757,7 @@ private static void PrepareFileData(string file) // Prepare csv raw data and write to temp files var rawDataRow = string.Join(",", s_colData) + "\n"; var rawData = string.Concat(Enumerable.Repeat(rawDataRow, NumberOfRows)); - + File.WriteAllText(file, rawData); t_filesToDelete.Add(file); } diff --git a/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs index a1c791071..08b85a9b5 100644 --- a/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs @@ -69,7 +69,6 @@ class SFAzureClientTest : SFBaseTest stageInfo = new PutGetStageInfo() { endPoint = EndPoint, - isClientSideEncrypted = true, location = Location, locationType = SFRemoteStorageUtil.AZURE_FS, path = LocationPath, diff --git a/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs index 0fad57542..d47742743 100644 --- a/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs @@ -62,7 +62,6 @@ class SFGCSClientTest : SFBaseTest stageInfo = new PutGetStageInfo() { endPoint = null, - isClientSideEncrypted = true, location = Location, locationType = SFRemoteStorageUtil.GCS_FS, path = LocationPath, diff --git a/Snowflake.Data.Tests/UnitTests/SFRemoteStorageClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFRemoteStorageClientTest.cs index 0e9d53767..76ec7c557 100644 --- a/Snowflake.Data.Tests/UnitTests/SFRemoteStorageClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFRemoteStorageClientTest.cs @@ -87,7 +87,6 @@ class SFRemoteStorageClientTest : SFBaseTest stageInfo = new PutGetStageInfo() { endPoint = EndPoint, - isClientSideEncrypted = true, location = Location, locationType = SFRemoteStorageUtil.GCS_FS, path = LocationPath, diff --git a/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs index 54647db8b..5432b0121 100644 --- a/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs @@ -87,7 +87,6 @@ class SFS3ClientTest : SFBaseTest stageInfo = new PutGetStageInfo() { endPoint = Endpoint, - isClientSideEncrypted = true, location = Location, locationType = SFRemoteStorageUtil.S3_FS, path = LocationPath, diff --git a/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs b/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs index b27daa51f..5d56b8db5 100644 --- a/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs +++ b/Snowflake.Data/Core/FileTransfer/SFFileTransferAgent.cs @@ -468,7 +468,7 @@ private void updatePresignedUrl() fileMeta.stageInfo = response.data.stageInfo; fileMeta.presignedUrl = response.data.stageInfo.presignedUrl; - } + } } else if (CommandTypes.DOWNLOAD == CommandType) { @@ -477,7 +477,7 @@ private void updatePresignedUrl() FilesMetas[index].presignedUrl = TransferMetadata.presignedUrls[index]; } } - } + } } /// @@ -544,7 +544,10 @@ private void initEncryptionMaterial() { if (CommandTypes.UPLOAD == CommandType) { - EncryptionMaterials.Add(TransferMetadata.encryptionMaterial[0]); + if (TransferMetadata.isClientSideEncrypted) + { + EncryptionMaterials.Add(TransferMetadata.encryptionMaterial[0]); + } } } @@ -670,7 +673,9 @@ private void initFileMetadata( overwrite = TransferMetadata.overwrite, presignedUrl = TransferMetadata.stageInfo.presignedUrl, parallel = TransferMetadata.parallel, - encryptionMaterial = TransferMetadata.encryptionMaterial[index], + encryptionMaterial = TransferMetadata.isClientSideEncrypted + ? TransferMetadata.encryptionMaterial[index] + : null, MaxBytesInMemory = GetFileTransferMaxBytesInMemory(), _operationType = CommandTypes.DOWNLOAD }; @@ -715,7 +720,7 @@ private int GetFileTransferMaxBytesInMemory() return FileTransferConfiguration.DefaultMaxBytesInMemory; } } - + /// /// Expand the wildcards if any to generate the list of paths for all files matched by the wildcards. /// Also replace the relative paths to the absolute paths for the files if needed. @@ -731,7 +736,7 @@ private List expandFileNames(string location) var directoryName = Path.GetDirectoryName(location); var foundDirectories = ExpandDirectories(directoryName); var filePaths = new List(); - + if (ContainsWildcard(fileName)) { foreach (var directory in foundDirectories) @@ -756,8 +761,8 @@ private List expandFileNames(string location) { filePaths.AddRange( Directory.GetFiles( - directory, - fileName, + directory, + fileName, SearchOption.TopDirectoryOnly)); } } @@ -788,7 +793,7 @@ private List expandFileNames(string location) return filePaths; } - + /// /// Expand the wildcards in the directory path to generate the list of directories to be searched for the files. /// @@ -803,7 +808,7 @@ private static IEnumerable ExpandDirectories(string directoryPath) { return new List { Path.GetFullPath(directoryPath) + Path.DirectorySeparatorChar }; } - + var pathParts = directoryPath.Split(Path.DirectorySeparatorChar); var resolvedPaths = new List(); @@ -863,7 +868,7 @@ private static IEnumerable ExpandDirectories(string directoryPath) private static string ExpandHomeDirectoryIfNeeded(string directoryPath) { if (!directoryPath.Contains('~')) return directoryPath; - + var homePath = (Environment.OSVersion.Platform == PlatformID.Unix || Environment.OSVersion.Platform == PlatformID.MacOSX) ? Environment.GetEnvironmentVariable("HOME") @@ -1036,7 +1041,7 @@ private async Task UploadFilesInSequentialAsync( { await updatePresignedUrlAsync(cancellationToken).ConfigureAwait(false); } - + // Break out of loop if file is successfully uploaded or already exists if (fileMetadata.resultStatus == ResultStatus.UPLOADED.ToString() || fileMetadata.resultStatus == ResultStatus.SKIPPED.ToString()) @@ -1429,7 +1434,7 @@ private void initFileMetadataForUpload() throw new ArgumentException("No file found for: " + TransferMetadata.src_locations[0].ToString()); } } - + private static bool IsDirectory(string path) { var attr = File.GetAttributes(path); diff --git a/Snowflake.Data/Core/RestResponse.cs b/Snowflake.Data/Core/RestResponse.cs index 64275fa42..47dafa2c4 100755 --- a/Snowflake.Data/Core/RestResponse.cs +++ b/Snowflake.Data/Core/RestResponse.cs @@ -382,6 +382,9 @@ internal class PutGetResponseData : IQueryExecResponseData [JsonProperty(PropertyName = "stageInfo", NullValueHandling = NullValueHandling.Ignore)] internal PutGetStageInfo stageInfo { get; set; } + [JsonProperty(PropertyName = "isClientSideEncrypted", NullValueHandling = NullValueHandling.Ignore)] + internal bool isClientSideEncrypted { get; set; } + [JsonProperty(PropertyName = "encryptionMaterial", NullValueHandling = NullValueHandling.Ignore)] [JsonConverter(typeof(SingleOrArrayConverter))] internal List encryptionMaterial { get; set; } @@ -428,9 +431,6 @@ internal class PutGetStageInfo [JsonProperty(PropertyName = "storageAccount", NullValueHandling = NullValueHandling.Ignore)] internal string storageAccount { get; set; } - [JsonProperty(PropertyName = "isClientSideEncrypted", NullValueHandling = NullValueHandling.Ignore)] - internal bool isClientSideEncrypted { get; set; } - [JsonProperty(PropertyName = "creds", NullValueHandling = NullValueHandling.Ignore)] internal Dictionary stageCredentials { get; set; }