From 4d3dff32ffd61312bb9160c2a3975246a32dde2f Mon Sep 17 00:00:00 2001 From: Dariusz Stempniak Date: Wed, 25 Oct 2023 15:17:35 +0200 Subject: [PATCH] SNOW-948463 Removed error information for PUT command for GCP --- .../IntegrationTests/SFPutGetTest.cs | 56 ++++++++++++++++++- .../UnitTests/SFGCSClientTest.cs | 8 ++- .../FileTransfer/StorageClient/SFGCSClient.cs | 11 ++-- .../StorageClient/SFSnowflakeAzureClient.cs | 2 +- 4 files changed, 66 insertions(+), 11 deletions(-) diff --git a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs index 382fabe04..7c5a2fdf2 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs @@ -253,6 +253,50 @@ public void TestPutFileRelativePathAsteriskWildcard() } } + [Test] + public void TestPutFileWithoutOverwriteFlagSkipsSecondUpload() + { + // Set the PUT query variables + t_inputFilePath = $"{Guid.NewGuid()}.csv"; + t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; + + PrepareFileData(t_inputFilePath); + + using (var conn = new SnowflakeDbConnection(ConnectionString)) + { + conn.Open(); + PutFile(conn, expectedStatus: ResultStatus.UPLOADED); + VerifyFilesAreUploaded(conn, new List { t_inputFilePath }, t_internalStagePath); + PutFile(conn, expectedStatus: ResultStatus.SKIPPED); + } + } + + [Test] + public void TestPutFileWithOverwriteFlagRunsSecondUpload() + { + var overwriteAttribute = "OVERWRITE=TRUE"; + + // Set the PUT query variables + t_inputFilePath = $"{Guid.NewGuid()}.csv"; + t_internalStagePath = $"@{t_schemaName}.{t_stageName}"; + + PrepareFileData(t_inputFilePath); + + using (var conn = new SnowflakeDbConnection(ConnectionString)) + { + conn.ConnectionString = ConnectionString; + + // for GCP force downscoped credential + if (Environment.GetEnvironmentVariable("snowflake_cloud_env") == "GCP") + conn.ConnectionString += "GCS_USE_DOWNSCOPED_CREDENTIAL=TRUE"; + + conn.Open(); + PutFile(conn, overwriteAttribute, expectedStatus: ResultStatus.UPLOADED); + VerifyFilesAreUploaded(conn, new List { t_inputFilePath }, t_internalStagePath); + PutFile(conn, overwriteAttribute, expectedStatus: ResultStatus.UPLOADED); + } + } + [Test] public void TestPutDirectoryAsteriskWildcard() { @@ -418,13 +462,18 @@ private static bool IsCompressedByTheDriver() } // PUT - upload file from local directory to the stage - private void PutFile(SnowflakeDbConnection conn) + void PutFile( + SnowflakeDbConnection conn, + String additionalAttribute = "", + ResultStatus expectedStatus = ResultStatus.UPLOADED) { using (var command = conn.CreateCommand()) { // Prepare PUT query string putQuery = - $"PUT file://{t_inputFilePath} {t_internalStagePath} AUTO_COMPRESS={(t_autoCompress ? "TRUE" : "FALSE")}"; + $"PUT file://{t_inputFilePath} {t_internalStagePath}" + + $" AUTO_COMPRESS={(t_autoCompress ? "TRUE" : "FALSE")}" + + $" {additionalAttribute}"; // Upload file command.CommandText = putQuery; @@ -432,7 +481,7 @@ private void PutFile(SnowflakeDbConnection conn) Assert.IsTrue(reader.Read()); // Check file status - Assert.AreEqual(ResultStatus.UPLOADED.ToString(), + Assert.AreEqual(expectedStatus.ToString(), reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.ResultStatus)); // Check source and destination compression type if (t_autoCompress) @@ -449,6 +498,7 @@ private void PutFile(SnowflakeDbConnection conn) Assert.AreEqual(SFFileCompressionTypes.NONE.Name, reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.DestinationCompressionType)); } + Assert.IsNull(reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.ErrorDetails)); } } diff --git a/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs index f894a82e6..da7c0cdf4 100644 --- a/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs @@ -201,12 +201,18 @@ private void AssertForGetFileHeaderTests(ResultStatus expectedResultStatus, File { Assert.AreEqual(MockGCSClient.ContentLength, fileHeader.contentLength); Assert.AreEqual(MockGCSClient.SFCDigest, fileHeader.digest); + Assert.IsNull(_fileMetadata.lastError); + } + else if (expectedResultStatus == ResultStatus.NOT_FOUND_FILE) + { + Assert.IsNull(fileHeader); + Assert.IsNull(_fileMetadata.lastError); } else { Assert.IsNull(fileHeader); + Assert.IsNotNull(_fileMetadata.lastError); } - Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus); } diff --git a/Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs b/Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs index 54326f670..14abaad4a 100644 --- a/Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs +++ b/Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs @@ -444,9 +444,7 @@ private void HandleDownloadResponse(HttpWebResponse response, SFFileMetadata fil private SFFileMetadata HandleFileHeaderErrForPresignedUrls(WebException ex, SFFileMetadata fileMetadata) { Logger.Error("Failed to get file header for presigned url: " + ex.Message); - - fileMetadata.lastError = ex; - + HttpWebResponse response = (HttpWebResponse)ex.Response; if (response.StatusCode == HttpStatusCode.Unauthorized || response.StatusCode == HttpStatusCode.Forbidden || @@ -457,6 +455,7 @@ private SFFileMetadata HandleFileHeaderErrForPresignedUrls(WebException ex, SFFi else { fileMetadata.resultStatus = ResultStatus.ERROR.ToString(); + fileMetadata.lastError = ex; } return fileMetadata; @@ -472,19 +471,18 @@ private SFFileMetadata HandleFileHeaderErrForGeneratedUrls(WebException ex, SFFi { Logger.Error("Failed to get file header for non-presigned url: " + ex.Message); - // If file doesn't exist, GET request fails - fileMetadata.lastError = ex; - HttpWebResponse response = (HttpWebResponse)ex.Response; if (response.StatusCode == HttpStatusCode.Unauthorized) { fileMetadata.resultStatus = ResultStatus.RENEW_TOKEN.ToString(); + fileMetadata.lastError = ex; } else if (response.StatusCode == HttpStatusCode.Forbidden || response.StatusCode == HttpStatusCode.InternalServerError || response.StatusCode == HttpStatusCode.ServiceUnavailable) { fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString(); + fileMetadata.lastError = ex; } else if (response.StatusCode == HttpStatusCode.NotFound) { @@ -493,6 +491,7 @@ private SFFileMetadata HandleFileHeaderErrForGeneratedUrls(WebException ex, SFFi else { fileMetadata.resultStatus = ResultStatus.ERROR.ToString(); + fileMetadata.lastError = ex; } return fileMetadata; } diff --git a/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs b/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs index c6834e0df..ff18c0672 100644 --- a/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs +++ b/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs @@ -191,7 +191,7 @@ public void UploadFile(SFFileMetadata fileMetadata, Stream fileBytesStream, SFEn { // Issue the POST/PUT request fileBytesStream.Position = 0; - blobClient.Upload(fileBytesStream); + blobClient.Upload(fileBytesStream, overwrite: true); blobClient.SetMetadata(metadata); } catch (RequestFailedException ex)