Skip to content

Commit

Permalink
SNOW-948463 Removed error information for PUT command for GCP (#801)
Browse files Browse the repository at this point in the history
### Description
SNOW-948463 Removed error information for PUT command for GCP

### Checklist
- [x] Code compiles correctly
- [x] Code is formatted according to [Coding
Conventions](../CodingConventions.md)
- [x] Created tests which fail without the change (if possible)
- [x] All tests passing (`dotnet test`)
- [x] Extended the README / documentation, if necessary
- [x] Provide JIRA issue id (if possible) or GitHub issue id in PR name
  • Loading branch information
sfc-gh-dstempniak authored Oct 27, 2023
1 parent eb0e25d commit 635b214
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 14 deletions.
53 changes: 50 additions & 3 deletions Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,47 @@ public void TestPutFileRelativePathAsteriskWildcard()
}
}

[Test]
// presigned url is enabled on CI so we need to disable the test
// it should be enabled when downscoped credential is the default option
[IgnoreOnEnvIs("snowflake_cloud_env", new [] { "GCP" })]
public void TestPutFileWithoutOverwriteFlagSkipsSecondUpload()
{
// Set the PUT query variables
t_inputFilePath = $"{Guid.NewGuid()}.csv";
t_internalStagePath = $"@{t_schemaName}.{t_stageName}";

PrepareFileData(t_inputFilePath);

using (var conn = new SnowflakeDbConnection(ConnectionString))
{
conn.Open();
PutFile(conn, expectedStatus: ResultStatus.UPLOADED);
VerifyFilesAreUploaded(conn, new List<string> { t_inputFilePath }, t_internalStagePath);
PutFile(conn, expectedStatus: ResultStatus.SKIPPED);
}
}

[Test]
public void TestPutFileWithOverwriteFlagRunsSecondUpload()
{
var overwriteAttribute = "OVERWRITE=TRUE";

// Set the PUT query variables
t_inputFilePath = $"{Guid.NewGuid()}.csv";
t_internalStagePath = $"@{t_schemaName}.{t_stageName}";

PrepareFileData(t_inputFilePath);

using (var conn = new SnowflakeDbConnection(ConnectionString))
{
conn.Open();
PutFile(conn, overwriteAttribute, expectedStatus: ResultStatus.UPLOADED);
VerifyFilesAreUploaded(conn, new List<string> { t_inputFilePath }, t_internalStagePath);
PutFile(conn, overwriteAttribute, expectedStatus: ResultStatus.UPLOADED);
}
}

[Test]
public void TestPutDirectoryAsteriskWildcard()
{
Expand Down Expand Up @@ -418,21 +459,26 @@ private static bool IsCompressedByTheDriver()
}

// PUT - upload file from local directory to the stage
private void PutFile(SnowflakeDbConnection conn)
void PutFile(
SnowflakeDbConnection conn,
String additionalAttribute = "",
ResultStatus expectedStatus = ResultStatus.UPLOADED)
{
using (var command = conn.CreateCommand())
{
// Prepare PUT query
string putQuery =
$"PUT file://{t_inputFilePath} {t_internalStagePath} AUTO_COMPRESS={(t_autoCompress ? "TRUE" : "FALSE")}";
$"PUT file://{t_inputFilePath} {t_internalStagePath}" +
$" AUTO_COMPRESS={(t_autoCompress ? "TRUE" : "FALSE")}" +
$" {additionalAttribute}";

// Upload file
command.CommandText = putQuery;
var reader = command.ExecuteReader();
Assert.IsTrue(reader.Read());

// Check file status
Assert.AreEqual(ResultStatus.UPLOADED.ToString(),
Assert.AreEqual(expectedStatus.ToString(),
reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.ResultStatus));
// Check source and destination compression type
if (t_autoCompress)
Expand All @@ -449,6 +495,7 @@ private void PutFile(SnowflakeDbConnection conn)
Assert.AreEqual(SFFileCompressionTypes.NONE.Name,
reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.DestinationCompressionType));
}
Assert.IsNull(reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.ErrorDetails));
}
}

Expand Down
4 changes: 2 additions & 2 deletions Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ public void TestUploadFile(HttpStatusCode httpStatusCode, ResultStatus expectedR
.Returns<string>((blobName) =>
{
var mockBlobClient = new Mock<BlobClient>();
mockBlobClient.Setup(client => client.Upload(It.IsAny<Stream>()))
mockBlobClient.Setup(client => client.Upload(It.IsAny<Stream>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
.Returns(() => MockAzureClient.createMockResponseForBlobContentInfo(key));

return mockBlobClient.Object;
Expand Down Expand Up @@ -251,7 +251,7 @@ public async Task TestUploadFileAsync(HttpStatusCode httpStatusCode, ResultStatu
.Returns<string>((blobName) =>
{
var mockBlobClient = new Mock<BlobClient>();
mockBlobClient.Setup(client => client.UploadAsync(It.IsAny<Stream>(), It.IsAny<CancellationToken>()))
mockBlobClient.Setup(client => client.UploadAsync(It.IsAny<Stream>(), It.IsAny<bool>(),It.IsAny<CancellationToken>()))
.Returns(async () => await Task.Run(() => MockAzureClient.createMockResponseForBlobContentInfo(key)).ConfigureAwait(false));

return mockBlobClient.Object;
Expand Down
8 changes: 7 additions & 1 deletion Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -201,12 +201,18 @@ private void AssertForGetFileHeaderTests(ResultStatus expectedResultStatus, File
{
Assert.AreEqual(MockGCSClient.ContentLength, fileHeader.contentLength);
Assert.AreEqual(MockGCSClient.SFCDigest, fileHeader.digest);
Assert.IsNull(_fileMetadata.lastError);
}
else if (expectedResultStatus == ResultStatus.NOT_FOUND_FILE)
{
Assert.IsNull(fileHeader);
Assert.IsNull(_fileMetadata.lastError);
}
else
{
Assert.IsNull(fileHeader);
Assert.IsNotNull(_fileMetadata.lastError);
}

Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus);
}

Expand Down
11 changes: 5 additions & 6 deletions Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -444,9 +444,7 @@ private void HandleDownloadResponse(HttpWebResponse response, SFFileMetadata fil
private SFFileMetadata HandleFileHeaderErrForPresignedUrls(WebException ex, SFFileMetadata fileMetadata)
{
Logger.Error("Failed to get file header for presigned url: " + ex.Message);

fileMetadata.lastError = ex;


HttpWebResponse response = (HttpWebResponse)ex.Response;
if (response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden ||
Expand All @@ -457,6 +455,7 @@ private SFFileMetadata HandleFileHeaderErrForPresignedUrls(WebException ex, SFFi
else
{
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
fileMetadata.lastError = ex;
}

return fileMetadata;
Expand All @@ -472,19 +471,18 @@ private SFFileMetadata HandleFileHeaderErrForGeneratedUrls(WebException ex, SFFi
{
Logger.Error("Failed to get file header for non-presigned url: " + ex.Message);

// If file doesn't exist, GET request fails
fileMetadata.lastError = ex;

HttpWebResponse response = (HttpWebResponse)ex.Response;
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
fileMetadata.resultStatus = ResultStatus.RENEW_TOKEN.ToString();
fileMetadata.lastError = ex;
}
else if (response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.InternalServerError ||
response.StatusCode == HttpStatusCode.ServiceUnavailable)
{
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
fileMetadata.lastError = ex;
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
Expand All @@ -493,6 +491,7 @@ private SFFileMetadata HandleFileHeaderErrForGeneratedUrls(WebException ex, SFFi
else
{
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
fileMetadata.lastError = ex;
}
return fileMetadata;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ public void UploadFile(SFFileMetadata fileMetadata, Stream fileBytesStream, SFEn
{
// Issue the POST/PUT request
fileBytesStream.Position = 0;
blobClient.Upload(fileBytesStream);
blobClient.Upload(fileBytesStream, overwrite: true);
blobClient.SetMetadata(metadata);
}
catch (RequestFailedException ex)
Expand Down Expand Up @@ -221,7 +221,7 @@ public async Task UploadFileAsync(SFFileMetadata fileMetadata, Stream fileBytesS
{
// Issue the POST/PUT request
fileBytesStream.Position = 0;
await blobClient.UploadAsync(fileBytesStream, cancellationToken).ConfigureAwait(false);
await blobClient.UploadAsync(fileBytesStream, true, cancellationToken).ConfigureAwait(false);
blobClient.SetMetadata(metadata);
}
catch (RequestFailedException ex)
Expand Down

0 comments on commit 635b214

Please sign in to comment.