SNOW-948463 Removed error information for PUT command for GCP

snowflakedb · Oct 25, 2023 · 4d3dff3 · 4d3dff3
1 parent 64b638c
commit 4d3dff3
Show file tree

Hide file tree

Showing 4 changed files with 66 additions and 11 deletions.
diff --git a/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs b/Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs
@@ -253,6 +253,50 @@ public void TestPutFileRelativePathAsteriskWildcard()
             }
         }
 
+        [Test]
+        public void TestPutFileWithoutOverwriteFlagSkipsSecondUpload()
+        {
+            // Set the PUT query variables
+            t_inputFilePath = $"{Guid.NewGuid()}.csv";
+            t_internalStagePath = $"@{t_schemaName}.{t_stageName}";
+
+            PrepareFileData(t_inputFilePath);
+
+            using (var conn = new SnowflakeDbConnection(ConnectionString))
+            {
+                conn.Open();
+                PutFile(conn, expectedStatus: ResultStatus.UPLOADED);
+                VerifyFilesAreUploaded(conn, new List<string> { t_inputFilePath }, t_internalStagePath);
+                PutFile(conn, expectedStatus: ResultStatus.SKIPPED);
+            }
+        }
+
+        [Test]
+        public void TestPutFileWithOverwriteFlagRunsSecondUpload()
+        {
+            var overwriteAttribute = "OVERWRITE=TRUE";
+
+            // Set the PUT query variables
+            t_inputFilePath = $"{Guid.NewGuid()}.csv";
+            t_internalStagePath = $"@{t_schemaName}.{t_stageName}";
+
+            PrepareFileData(t_inputFilePath);
+
+            using (var conn = new SnowflakeDbConnection(ConnectionString))
+            {
+                conn.ConnectionString = ConnectionString;
+
+                // for GCP force downscoped credential
+                if (Environment.GetEnvironmentVariable("snowflake_cloud_env") == "GCP")
+                    conn.ConnectionString += "GCS_USE_DOWNSCOPED_CREDENTIAL=TRUE";
+
+                conn.Open();
+                PutFile(conn, overwriteAttribute, expectedStatus: ResultStatus.UPLOADED);
+                VerifyFilesAreUploaded(conn, new List<string> { t_inputFilePath }, t_internalStagePath);
+                PutFile(conn, overwriteAttribute, expectedStatus: ResultStatus.UPLOADED);
+            }
+        }
+
         [Test]
         public void TestPutDirectoryAsteriskWildcard()
         {
@@ -418,21 +462,26 @@ private static bool IsCompressedByTheDriver()
         }
 
         // PUT - upload file from local directory to the stage
-        private void PutFile(SnowflakeDbConnection conn)
+        void PutFile(
+            SnowflakeDbConnection conn, 
+            String additionalAttribute = "", 
+            ResultStatus expectedStatus = ResultStatus.UPLOADED)
         {
             using (var command = conn.CreateCommand())
             {
                 // Prepare PUT query
                 string putQuery =
-                    $"PUT file://{t_inputFilePath} {t_internalStagePath} AUTO_COMPRESS={(t_autoCompress ? "TRUE" : "FALSE")}";
+                    $"PUT file://{t_inputFilePath} {t_internalStagePath}" +
+                    $" AUTO_COMPRESS={(t_autoCompress ? "TRUE" : "FALSE")}" +
+                    $" {additionalAttribute}";
 
                 // Upload file
                 command.CommandText = putQuery;
                 var reader = command.ExecuteReader();
                 Assert.IsTrue(reader.Read());
 
                 // Check file status
-                Assert.AreEqual(ResultStatus.UPLOADED.ToString(),
+                Assert.AreEqual(expectedStatus.ToString(),
                     reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.ResultStatus));
                 // Check source and destination compression type
                 if (t_autoCompress)
@@ -449,6 +498,7 @@ private void PutFile(SnowflakeDbConnection conn)
                     Assert.AreEqual(SFFileCompressionTypes.NONE.Name,
                         reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.DestinationCompressionType));
                 }
+                Assert.IsNull(reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.ErrorDetails));
             }
         }
 

diff --git a/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs
@@ -201,12 +201,18 @@ private void AssertForGetFileHeaderTests(ResultStatus expectedResultStatus, File
             {
                 Assert.AreEqual(MockGCSClient.ContentLength, fileHeader.contentLength);
                 Assert.AreEqual(MockGCSClient.SFCDigest, fileHeader.digest);
+                Assert.IsNull(_fileMetadata.lastError);
+            }
+            else if (expectedResultStatus == ResultStatus.NOT_FOUND_FILE)
+            {
+                Assert.IsNull(fileHeader);
+                Assert.IsNull(_fileMetadata.lastError);
             }
             else
             {
                 Assert.IsNull(fileHeader);
+                Assert.IsNotNull(_fileMetadata.lastError);
             }
-
             Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus);
         }
 

diff --git a/Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs b/Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs
@@ -444,9 +444,7 @@ private void HandleDownloadResponse(HttpWebResponse response, SFFileMetadata fil
         private SFFileMetadata HandleFileHeaderErrForPresignedUrls(WebException ex, SFFileMetadata fileMetadata)
         {
             Logger.Error("Failed to get file header for presigned url: " + ex.Message);
-
-            fileMetadata.lastError = ex;
-
+
             HttpWebResponse response = (HttpWebResponse)ex.Response;
             if (response.StatusCode == HttpStatusCode.Unauthorized ||
                 response.StatusCode == HttpStatusCode.Forbidden ||
@@ -457,6 +455,7 @@ private SFFileMetadata HandleFileHeaderErrForPresignedUrls(WebException ex, SFFi
             else
             {
                 fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
+                fileMetadata.lastError = ex;
             }
 
             return fileMetadata;
@@ -472,19 +471,18 @@ private SFFileMetadata HandleFileHeaderErrForGeneratedUrls(WebException ex, SFFi
         {
             Logger.Error("Failed to get file header for non-presigned url: " + ex.Message);
 
-            // If file doesn't exist, GET request fails
-            fileMetadata.lastError = ex;
-
             HttpWebResponse response = (HttpWebResponse)ex.Response;
             if (response.StatusCode == HttpStatusCode.Unauthorized)
             {
                 fileMetadata.resultStatus = ResultStatus.RENEW_TOKEN.ToString();
+                fileMetadata.lastError = ex;
             }
             else if (response.StatusCode == HttpStatusCode.Forbidden ||
                 response.StatusCode == HttpStatusCode.InternalServerError ||
                 response.StatusCode == HttpStatusCode.ServiceUnavailable)
             {
                 fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
+                fileMetadata.lastError = ex;
             }
             else if (response.StatusCode == HttpStatusCode.NotFound)
             {
@@ -493,6 +491,7 @@ private SFFileMetadata HandleFileHeaderErrForGeneratedUrls(WebException ex, SFFi
             else
             {
                 fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
+                fileMetadata.lastError = ex;
             }
             return fileMetadata;
         }

diff --git a/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs b/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs
@@ -191,7 +191,7 @@ public void UploadFile(SFFileMetadata fileMetadata, Stream fileBytesStream, SFEn
             {
                 // Issue the POST/PUT request
                 fileBytesStream.Position = 0;
-                blobClient.Upload(fileBytesStream);
+                blobClient.Upload(fileBytesStream, overwrite: true);
                 blobClient.SetMetadata(metadata);
             }
             catch (RequestFailedException ex)