Skip to content

Commit

Permalink
Fix after review
Browse files Browse the repository at this point in the history
  • Loading branch information
@sfc-gh-dprzybysz
sfc-gh-dprzybysz committed Nov 25, 2024
1 parent 1793940 commit fdd505f
Show file tree
Hide file tree
Showing 9 changed files with 354 additions and 346 deletions.
1 change: 1 addition & 0 deletions src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1116,6 +1116,7 @@ static StageInfo getStageInfo(JsonNode jsonNode, SFSession session) throws Snowf
|| "GCS".equalsIgnoreCase(stageLocationType)) {
endPoint = jsonNode.path("data").path("stageInfo").findValue("endPoint").asText();
if ("GCS".equalsIgnoreCase(stageLocationType)
&& endPoint != null
&& (endPoint.trim().isEmpty() || "null".equals(endPoint))) {
// setting to null to preserve previous behaviour for GCS
endPoint = null;
Expand Down
375 changes: 34 additions & 341 deletions src/test/java/net/snowflake/client/jdbc/FileUploaderPrep.java
View file

Large diffs are not rendered by default.

11 changes: 6 additions & 5 deletions src/test/java/net/snowflake/client/jdbc/FileUploaderSessionlessTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
Expand Down Expand Up @@ -320,7 +321,7 @@ public void testGetFileTransferMetadatasUploadError() throws Exception {
JsonNode downloadNode = mapper.readTree("{\"data\": {\"command\": \"DOWNLOAD\"}}");
try {
SnowflakeFileTransferAgent.getFileTransferMetadatas(downloadNode);
assertTrue(false);
fail();
} catch (SnowflakeSQLException err) {
Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode());
Assert.assertEquals(
Expand All @@ -333,7 +334,7 @@ public void testGetFileTransferMetadatasEncryptionMaterialError() throws Excepti
JsonNode garbageNode = mapper.readTree("{\"data\": {\"src_locations\": [1, 2]}}");
try {
SnowflakeFileTransferAgent.getFileTransferMetadatas(garbageNode);
assertTrue(false);
fail();
} catch (SnowflakeSQLException err) {
Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode());
assertTrue(
Expand All @@ -348,7 +349,7 @@ public void testGetFileTransferMetadatasUnsupportedLocationError() throws Except
foo.put("locationType", "LOCAL_FS");
try {
SnowflakeFileTransferAgent.getFileTransferMetadatas(modifiedNode);
assertTrue(false);
fail();
} catch (SnowflakeSQLException err) {
Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode());
assertTrue(err.getMessage().contains("JDBC driver internal error: This API only supports"));
Expand All @@ -360,7 +361,7 @@ public void testGetFileTransferMetadatasSrcLocationsArrayError() throws JsonProc
JsonNode garbageNode = mapper.readTree("{\"data\": {\"src_locations\": \"abc\"}}");
try {
SnowflakeFileTransferAgent.getFileTransferMetadatas(garbageNode);
assertTrue(false);
fail();
} catch (SnowflakeSQLException err) {
Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode());
assertTrue(
Expand All @@ -375,7 +376,7 @@ public void testGetFileMetadatasEncryptionMaterialsException() {
foo.put("encryptionMaterial", "[1, 2, 3]]");
try {
SnowflakeFileTransferAgent.getFileTransferMetadatas(modifiedNode);
assertTrue(false);
fail();
} catch (SnowflakeSQLException err) {
Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode());
assertTrue(err.getMessage().contains("Failed to parse encryptionMaterial"));
Expand Down
51 changes: 51 additions & 0 deletions src/test/resources/FileUploaderPrep/exampleAzure.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
{
"data": {
"uploadInfo": {
"locationType": "AZURE",
"location": "EXAMPLE_LOCATION/",
"path": "EXAMPLE_PATH/",
"region": "westus",
"storageAccount": "sfcdev2stage",
"isClientSideEncrypted": true,
"creds": {
"AZURE_SAS_TOKEN": "EXAMPLE_AZURE_SAS_TOKEN"
},
"presignedUrl": null,
"endPoint": "blob.core.windows.net"
},
"src_locations": [
"/foo/orders_100.csv"
],
"parallel": 4,
"threshold": 209715200,
"autoCompress": true,
"overwrite": false,
"sourceCompression": "auto_detect",
"clientShowEncryptionParameter": false,
"queryId": "EXAMPLE_QUERY_ID",
"encryptionMaterial": {
"queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY",
"queryId": "EXAMPLE_QUERY_ID",
"smkId": 123
},
"stageInfo": {
"locationType": "AZURE",
"location": "EXAMPLE_LOCATION/",
"path": "EXAMPLE_PATH/",
"region": "westus",
"storageAccount": "EXAMPLE_STORAGE_ACCOUNT",
"isClientSideEncrypted": true,
"creds": {
"AZURE_SAS_TOKEN": "EXAMPLE_AZURE_SAS_TOKEN"
},
"presignedUrl": null,
"endPoint": "blob.core.windows.net"
},
"command": "UPLOAD",
"kind": null,
"operation": "Node"
},
"code": null,
"message": null,
"success": true
}
47 changes: 47 additions & 0 deletions src/test/resources/FileUploaderPrep/exampleGCS.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
{
"data": {
"uploadInfo": {
"locationType": "GCS",
"location": "foo/tables/9224/",
"path": "tables/9224/",
"region": "US-WEST1",
"storageAccount": "",
"isClientSideEncrypted": true,
"creds": {},
"presignedUrl": "EXAMPLE_PRESIGNED_URL",
"endPoint": ""
},
"src_locations": [
"/foo/bart/orders_100.csv"
],
"parallel": 4,
"threshold": 209715200,
"autoCompress": true,
"overwrite": false,
"sourceCompression": "auto_detect",
"clientShowEncryptionParameter": false,
"queryId": "EXAMPLE_QUERY_ID",
"encryptionMaterial": {
"queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY",
"queryId": "EXAMPLE_QUERY_ID",
"smkId": 123
},
"stageInfo": {
"locationType": "GCS",
"location": "foo/tables/9224/",
"path": "tables/9224/",
"region": "US-WEST1",
"storageAccount": "",
"isClientSideEncrypted": true,
"creds": {},
"presignedUrl": "EXAMPLE_PRESIGNED_URL",
"endPoint": ""
},
"command": "UPLOAD",
"kind": null,
"operation": "Node"
},
"code": null,
"message": null,
"success": true
}
47 changes: 47 additions & 0 deletions src/test/resources/FileUploaderPrep/exampleGCSWithEndpoint.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
{
"data": {
"uploadInfo": {
"locationType": "GCS",
"location": "foo/tables/9224/",
"path": "tables/9224/",
"region": "US-WEST1",
"storageAccount": "",
"isClientSideEncrypted": true,
"creds": {},
"presignedUrl": "EXAMPLE_PRESIGNED_URL",
"endPoint": "example.com"
},
"src_locations": [
"/foo/bart/orders_100.csv"
],
"parallel": 4,
"threshold": 209715200,
"autoCompress": true,
"overwrite": false,
"sourceCompression": "auto_detect",
"clientShowEncryptionParameter": false,
"queryId": "EXAMPLE_QUERY_ID",
"encryptionMaterial": {
"queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY",
"queryId": "EXAMPLE_QUERY_ID",
"smkId": 123
},
"stageInfo": {
"locationType": "GCS",
"location": "foo/tables/9224/",
"path": "tables/9224/",
"region": "US-WEST1",
"storageAccount": "",
"isClientSideEncrypted": true,
"creds": {},
"presignedUrl": "EXAMPLE_PRESIGNED_URL",
"endPoint": "example.com"
},
"command": "UPLOAD",
"kind": null,
"operation": "Node"
},
"code": null,
"message": null,
"success": true
}
49 changes: 49 additions & 0 deletions src/test/resources/FileUploaderPrep/exampleGCSWithUseRegionalUrl.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
{
"data": {
"uploadInfo": {
"locationType": "GCS",
"useRegionalUrl": true,
"location": "foo/tables/9224/",
"path": "tables/9224/",
"region": "US-WEST1",
"storageAccount": "",
"isClientSideEncrypted": true,
"creds": {},
"presignedUrl": "EXAMPLE_PRESIGNED_URL",
"endPoint": ""
},
"src_locations": [
"/foo/bart/orders_100.csv"
],
"parallel": 4,
"threshold": 209715200,
"autoCompress": true,
"overwrite": false,
"sourceCompression": "auto_detect",
"clientShowEncryptionParameter": false,
"queryId": "EXAMPLE_QUERY_ID",
"encryptionMaterial": {
"queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY",
"queryId": "EXAMPLE_QUERY_ID",
"smkId": 123
},
"stageInfo": {
"locationType": "GCS",
"useRegionalUrl": true,
"location": "foo/tables/9224/",
"path": "tables/9224/",
"region": "US-WEST1",
"storageAccount": "",
"isClientSideEncrypted": true,
"creds": {},
"presignedUrl": "EXAMPLE_PRESIGNED_URL",
"endPoint": ""
},
"command": "UPLOAD",
"kind": null,
"operation": "Node"
},
"code": null,
"message": null,
"success": true
}
60 changes: 60 additions & 0 deletions src/test/resources/FileUploaderPrep/exampleS3.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
{
"data": {
"uploadInfo": {
"locationType": "S3",
"location": "example/location",
"path": "tables/19805757505/",
"region": "us-west-2",
"storageAccount": null,
"isClientSideEncrypted": true,
"creds": {
"AWS_KEY_ID": "EXAMPLE_AWS_KEY_ID",
"AWS_SECRET_KEY": "EXAMPLE_AWS_SECRET_KEY",
"AWS_TOKEN": "EXAMPLE_AWS_TOKEN",
"AWS_ID": "EXAMPLE_AWS_ID",
"AWS_KEY": "EXAMPLE_AWS_KEY"
},
"presignedUrl": null,
"endPoint": null
},
"src_locations": [
"/tmp/files/orders_100.csv"
],
"parallel": 4,
"threshold": 209715200,
"autoCompress": true,
"overwrite": false,
"sourceCompression": "auto_detect",
"clientShowEncryptionParameter": true,
"queryId": "EXAMPLE_QUERY_ID",
"encryptionMaterial": {
"queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY",
"queryId": "EXAMPLE_QUERY_ID",
"smkId": 123
},
"stageInfo": {
"locationType": "S3",
"location": "stage/location/foo/",
"path": "tables/19805757505/",
"region": "us-west-2",
"storageAccount": null,
"isClientSideEncrypted": true,
"useS3RegionalUrl": true,
"creds": {
"AWS_KEY_ID": "EXAMPLE_AWS_KEY_ID",
"AWS_SECRET_KEY": "EXAMPLE_AWS_SECRET_KEY",
"AWS_TOKEN": "EXAMPLE_AWS_TOKEN",
"AWS_ID": "EXAMPLE_AWS_ID",
"AWS_KEY": "EXAMPLE_AWS_KEY"
},
"presignedUrl": null,
"endPoint": null
},
"command": "UPLOAD",
"kind": null,
"operation": "Node"
},
"code": null,
"message": null,
"success": true
}
59 changes: 59 additions & 0 deletions src/test/resources/FileUploaderPrep/exampleS3WithStageEndpoint.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
{
"data": {
"uploadInfo": {
"locationType": "S3",
"location": "example/location",
"path": "tables/19805757505/",
"region": "us-west-2",
"storageAccount": null,
"isClientSideEncrypted": true,
"creds": {
"AWS_KEY_ID": "EXAMPLE_AWS_KEY_ID",
"AWS_SECRET_KEY": "EXAMPLE_AWS_SECRET_KEY",
"AWS_TOKEN": "EXAMPLE_AWS_TOKEN",
"AWS_ID": "EXAMPLE_AWS_ID",
"AWS_KEY": "EXAMPLE_AWS_KEY"
},
"presignedUrl": null,
"endPoint": null
},
"src_locations": [
"/tmp/files/orders_100.csv"
],
"parallel": 4,
"threshold": 209715200,
"autoCompress": true,
"overwrite": false,
"sourceCompression": "auto_detect",
"clientShowEncryptionParameter": true,
"queryId": "EXAMPLE_QUERY_ID",
"encryptionMaterial": {
"queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY",
"queryId": "EXAMPLE_QUERY_ID",
"smkId": 123
},
"stageInfo": {
"locationType": "S3",
"location": "stage/location/foo/",
"path": "tables/19805757505/",
"region": "us-west-2",
"storageAccount": null,
"isClientSideEncrypted": true,
"creds": {
"AWS_KEY_ID": "EXAMPLE_AWS_KEY_ID",
"AWS_SECRET_KEY": "EXAMPLE_AWS_SECRET_KEY",
"AWS_TOKEN": "EXAMPLE_AWS_TOKEN",
"AWS_ID": "EXAMPLE_AWS_ID",
"AWS_KEY": "EXAMPLE_AWS_KEY"
},
"presignedUrl": null,
"endPoint": "s3-fips.us-east-1.amazonaws.com"
},
"command": "UPLOAD",
"kind": null,
"operation": "Node"
},
"code": null,
"message": null,
"success": true
}

0 comments on commit fdd505f

Please sign in to comment.