Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SNOW-948463 Removed error information for PUT command for GCP #801

Merged
merged 1 commit into from
Oct 27, 2023
Merged

SNOW-948463 Removed error information for PUT command for GCP #801

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 50 additions & 3 deletions Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,47 @@ public void TestPutFileRelativePathAsteriskWildcard()
}
}

[Test]
// presigned url is enabled on CI so we need to disable the test
// it should be enabled when downscoped credential is the default option
[IgnoreOnEnvIs("snowflake_cloud_env", new [] { "GCP" })]
public void TestPutFileWithoutOverwriteFlagSkipsSecondUpload()
{
// Set the PUT query variables
t_inputFilePath = $"{Guid.NewGuid()}.csv";
t_internalStagePath = $"@{t_schemaName}.{t_stageName}";

PrepareFileData(t_inputFilePath);

using (var conn = new SnowflakeDbConnection(ConnectionString))
{
conn.Open();
PutFile(conn, expectedStatus: ResultStatus.UPLOADED);
VerifyFilesAreUploaded(conn, new List<string> { t_inputFilePath }, t_internalStagePath);
PutFile(conn, expectedStatus: ResultStatus.SKIPPED);
}
}

[Test]
public void TestPutFileWithOverwriteFlagRunsSecondUpload()
{
var overwriteAttribute = "OVERWRITE=TRUE";

// Set the PUT query variables
t_inputFilePath = $"{Guid.NewGuid()}.csv";
t_internalStagePath = $"@{t_schemaName}.{t_stageName}";

PrepareFileData(t_inputFilePath);

using (var conn = new SnowflakeDbConnection(ConnectionString))
{
conn.Open();
PutFile(conn, overwriteAttribute, expectedStatus: ResultStatus.UPLOADED);
VerifyFilesAreUploaded(conn, new List<string> { t_inputFilePath }, t_internalStagePath);
PutFile(conn, overwriteAttribute, expectedStatus: ResultStatus.UPLOADED);
}
}

[Test]
public void TestPutDirectoryAsteriskWildcard()
{
Expand Down Expand Up @@ -418,21 +459,26 @@ private static bool IsCompressedByTheDriver()
}

// PUT - upload file from local directory to the stage
private void PutFile(SnowflakeDbConnection conn)
void PutFile(
SnowflakeDbConnection conn,
String additionalAttribute = "",
ResultStatus expectedStatus = ResultStatus.UPLOADED)
{
using (var command = conn.CreateCommand())
{
// Prepare PUT query
string putQuery =
$"PUT file://{t_inputFilePath} {t_internalStagePath} AUTO_COMPRESS={(t_autoCompress ? "TRUE" : "FALSE")}";
$"PUT file://{t_inputFilePath} {t_internalStagePath}" +
$" AUTO_COMPRESS={(t_autoCompress ? "TRUE" : "FALSE")}" +
$" {additionalAttribute}";

// Upload file
command.CommandText = putQuery;
var reader = command.ExecuteReader();
Assert.IsTrue(reader.Read());

// Check file status
Assert.AreEqual(ResultStatus.UPLOADED.ToString(),
Assert.AreEqual(expectedStatus.ToString(),
reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.ResultStatus));
// Check source and destination compression type
if (t_autoCompress)
Expand All @@ -449,6 +495,7 @@ private void PutFile(SnowflakeDbConnection conn)
Assert.AreEqual(SFFileCompressionTypes.NONE.Name,
reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.DestinationCompressionType));
}
Assert.IsNull(reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.ErrorDetails));
}
}

Expand Down
4 changes: 2 additions & 2 deletions Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
SFFileMetadata _fileMetadata;

[SetUp]
public void BeforeTest()

Check warning on line 63 in Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, GCP) 

'SFAzureClientTest.BeforeTest()' hides inherited member 'SFBaseTestAsync.BeforeTest()'. Use the new keyword if hiding was intended.

Check warning on line 63 in Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AZURE) 

'SFAzureClientTest.BeforeTest()' hides inherited member 'SFBaseTestAsync.BeforeTest()'. Use the new keyword if hiding was intended.

Check warning on line 63 in Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, GCP) 

'SFAzureClientTest.BeforeTest()' hides inherited member 'SFBaseTestAsync.BeforeTest()'. Use the new keyword if hiding was intended.

Check warning on line 63 in Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AWS) 

'SFAzureClientTest.BeforeTest()' hides inherited member 'SFBaseTestAsync.BeforeTest()'. Use the new keyword if hiding was intended.

Check warning on line 63 in Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AWS) 

'SFAzureClientTest.BeforeTest()' hides inherited member 'SFBaseTestAsync.BeforeTest()'. Use the new keyword if hiding was intended.

Check warning on line 63 in Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AZURE) 

'SFAzureClientTest.BeforeTest()' hides inherited member 'SFBaseTestAsync.BeforeTest()'. Use the new keyword if hiding was intended.
{
t_downloadFileName = TestNameWithWorker + "_mockFileName.txt";

Expand Down Expand Up @@ -207,7 +207,7 @@
.Returns<string>((blobName) =>
{
var mockBlobClient = new Mock<BlobClient>();
mockBlobClient.Setup(client => client.Upload(It.IsAny<Stream>()))
mockBlobClient.Setup(client => client.Upload(It.IsAny<Stream>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
.Returns(() => MockAzureClient.createMockResponseForBlobContentInfo(key));

return mockBlobClient.Object;
Expand Down Expand Up @@ -251,7 +251,7 @@
.Returns<string>((blobName) =>
{
var mockBlobClient = new Mock<BlobClient>();
mockBlobClient.Setup(client => client.UploadAsync(It.IsAny<Stream>(), It.IsAny<CancellationToken>()))
mockBlobClient.Setup(client => client.UploadAsync(It.IsAny<Stream>(), It.IsAny<bool>(),It.IsAny<CancellationToken>()))
.Returns(async () => await Task.Run(() => MockAzureClient.createMockResponseForBlobContentInfo(key)).ConfigureAwait(false));

return mockBlobClient.Object;
Expand Down
8 changes: 7 additions & 1 deletion Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
SFFileMetadata _fileMetadata;

[SetUp]
public void BeforeTest()

Check warning on line 56 in Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, GCP) 

'SFGCSClientTest.BeforeTest()' hides inherited member 'SFBaseTestAsync.BeforeTest()'. Use the new keyword if hiding was intended.

Check warning on line 56 in Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AZURE) 

'SFGCSClientTest.BeforeTest()' hides inherited member 'SFBaseTestAsync.BeforeTest()'. Use the new keyword if hiding was intended.

Check warning on line 56 in Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, GCP) 

'SFGCSClientTest.BeforeTest()' hides inherited member 'SFBaseTestAsync.BeforeTest()'. Use the new keyword if hiding was intended.

Check warning on line 56 in Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net471, AWS) 

'SFGCSClientTest.BeforeTest()' hides inherited member 'SFBaseTestAsync.BeforeTest()'. Use the new keyword if hiding was intended.

Check warning on line 56 in Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AWS) 

'SFGCSClientTest.BeforeTest()' hides inherited member 'SFBaseTestAsync.BeforeTest()'. Use the new keyword if hiding was intended.

Check warning on line 56 in Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net472, AZURE) 

'SFGCSClientTest.BeforeTest()' hides inherited member 'SFBaseTestAsync.BeforeTest()'. Use the new keyword if hiding was intended.
{
t_downloadFileName = TestNameWithWorker + "_mockFileName.txt";

Expand Down Expand Up @@ -201,12 +201,18 @@
{
Assert.AreEqual(MockGCSClient.ContentLength, fileHeader.contentLength);
Assert.AreEqual(MockGCSClient.SFCDigest, fileHeader.digest);
Assert.IsNull(_fileMetadata.lastError);
}
else if (expectedResultStatus == ResultStatus.NOT_FOUND_FILE)
{
Assert.IsNull(fileHeader);
Assert.IsNull(_fileMetadata.lastError);
}
else
{
Assert.IsNull(fileHeader);
Assert.IsNotNull(_fileMetadata.lastError);
}

Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus);
}

Expand Down
11 changes: 5 additions & 6 deletions Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@
generateFileURL(fileMetadata.stageInfo.location, fileMetadata.RemoteFileName()) :
fileMetadata.presignedUrl;

WebRequest request = WebRequest.Create(url);

Check warning on line 118 in Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs

View workflow job for this annotation

GitHub Actions / Tests on Linux (net6.0, AZURE) 

'WebRequest.Create(string)' is obsolete: 'WebRequest, HttpWebRequest, ServicePoint, and WebClient are obsolete. Use HttpClient instead.' (https://aka.ms/dotnet-warnings/SYSLIB0014)

Check warning on line 118 in Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs

View workflow job for this annotation

GitHub Actions / Tests on Linux (net6.0, GCP) 

'WebRequest.Create(string)' is obsolete: 'WebRequest, HttpWebRequest, ServicePoint, and WebClient are obsolete. Use HttpClient instead.' (https://aka.ms/dotnet-warnings/SYSLIB0014)

Check warning on line 118 in Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs

View workflow job for this annotation

GitHub Actions / Tests on MAC (net6.0, GCP) 

'WebRequest.Create(string)' is obsolete: 'WebRequest, HttpWebRequest, ServicePoint, and WebClient are obsolete. Use HttpClient instead.' (https://aka.ms/dotnet-warnings/SYSLIB0014)

Check warning on line 118 in Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs

View workflow job for this annotation

GitHub Actions / Tests on Linux (net6.0, AWS) 

'WebRequest.Create(string)' is obsolete: 'WebRequest, HttpWebRequest, ServicePoint, and WebClient are obsolete. Use HttpClient instead.' (https://aka.ms/dotnet-warnings/SYSLIB0014)

Check warning on line 118 in Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net6.0, AZURE) 

'WebRequest.Create(string)' is obsolete: 'WebRequest, HttpWebRequest, ServicePoint, and WebClient are obsolete. Use HttpClient instead.' (https://aka.ms/dotnet-warnings/SYSLIB0014)

Check warning on line 118 in Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs

View workflow job for this annotation

GitHub Actions / Tests on Windows (net6.0, GCP) 

'WebRequest.Create(string)' is obsolete: 'WebRequest, HttpWebRequest, ServicePoint, and WebClient are obsolete. Use HttpClient instead.' (https://aka.ms/dotnet-warnings/SYSLIB0014)
request.Headers.Add("Authorization", $"Bearer {AccessToken}");
request.Method = method;

Expand Down Expand Up @@ -444,9 +444,7 @@
private SFFileMetadata HandleFileHeaderErrForPresignedUrls(WebException ex, SFFileMetadata fileMetadata)
{
Logger.Error("Failed to get file header for presigned url: " + ex.Message);

fileMetadata.lastError = ex;


HttpWebResponse response = (HttpWebResponse)ex.Response;
if (response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden ||
Expand All @@ -457,6 +455,7 @@
else
{
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
fileMetadata.lastError = ex;

Check warning on line 458 in Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs

View check run for this annotation

Codecov / codecov/patch

Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs#L458 

Added line #L458 was not covered by tests
}

return fileMetadata;
Expand All @@ -472,19 +471,18 @@
{
Logger.Error("Failed to get file header for non-presigned url: " + ex.Message);

// If file doesn't exist, GET request fails
fileMetadata.lastError = ex;
sfc-gh-knozderko marked this conversation as resolved.
Show resolved Hide resolved

HttpWebResponse response = (HttpWebResponse)ex.Response;
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
fileMetadata.resultStatus = ResultStatus.RENEW_TOKEN.ToString();
fileMetadata.lastError = ex;
}
else if (response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.InternalServerError ||
response.StatusCode == HttpStatusCode.ServiceUnavailable)
{
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
fileMetadata.lastError = ex;
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
Expand All @@ -493,6 +491,7 @@
else
{
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
fileMetadata.lastError = ex;
}
return fileMetadata;
}
Expand Down
4 changes: 2 additions & 2 deletions Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ public void UploadFile(SFFileMetadata fileMetadata, Stream fileBytesStream, SFEn
{
// Issue the POST/PUT request
fileBytesStream.Position = 0;
blobClient.Upload(fileBytesStream);
blobClient.Upload(fileBytesStream, overwrite: true);
blobClient.SetMetadata(metadata);
}
catch (RequestFailedException ex)
Expand Down Expand Up @@ -221,7 +221,7 @@ public async Task UploadFileAsync(SFFileMetadata fileMetadata, Stream fileBytesS
{
// Issue the POST/PUT request
fileBytesStream.Position = 0;
await blobClient.UploadAsync(fileBytesStream, cancellationToken).ConfigureAwait(false);
await blobClient.UploadAsync(fileBytesStream, true, cancellationToken).ConfigureAwait(false);
blobClient.SetMetadata(metadata);
}
catch (RequestFailedException ex)
Expand Down
Loading