diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java index a573a7466..4213b33b0 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java @@ -1116,6 +1116,7 @@ static StageInfo getStageInfo(JsonNode jsonNode, SFSession session) throws Snowf || "GCS".equalsIgnoreCase(stageLocationType)) { endPoint = jsonNode.path("data").path("stageInfo").findValue("endPoint").asText(); if ("GCS".equalsIgnoreCase(stageLocationType) + && endPoint != null && (endPoint.trim().isEmpty() || "null".equals(endPoint))) { // setting to null to preserve previous behaviour for GCS endPoint = null; diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderPrep.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderPrep.java index 90eb76866..d801a00ac 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderPrep.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderPrep.java @@ -6,356 +6,49 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; import java.util.List; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TemporaryFolder; /** File uploader test prep reused by IT/connection tests and sessionless tests */ abstract class FileUploaderPrep extends BaseJDBCTest { @Rule public TemporaryFolder folder = new TemporaryFolder(); - private ObjectMapper mapper = new ObjectMapper(); - private final String exampleS3JsonStringWithStageEndpoint = - "{\n" - + " \"data\": {\n" - + " \"uploadInfo\": {\n" - + " \"locationType\": \"S3\",\n" - + " \"location\": \"example/location\",\n" - + " \"path\": \"tables/19805757505/\",\n" - + " \"region\": \"us-west-2\",\n" - + " \"storageAccount\": null,\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {\n" - + " \"AWS_KEY_ID\": \"EXAMPLE_AWS_KEY_ID\",\n" - + " \"AWS_SECRET_KEY\": \"EXAMPLE_AWS_SECRET_KEY\",\n" - + " \"AWS_TOKEN\": \"EXAMPLE_AWS_TOKEN\",\n" - + " \"AWS_ID\": \"EXAMPLE_AWS_ID\",\n" - + " \"AWS_KEY\": \"EXAMPLE_AWS_KEY\"\n" - + " },\n" - + " \"presignedUrl\": null,\n" - + " \"endPoint\": null\n" - + " },\n" - + " \"src_locations\": [\n" - + " \"/tmp/files/orders_100.csv\"\n" - + " ],\n" - + " \"parallel\": 4,\n" - + " \"threshold\": 209715200,\n" - + " \"autoCompress\": true,\n" - + " \"overwrite\": false,\n" - + " \"sourceCompression\": \"auto_detect\",\n" - + " \"clientShowEncryptionParameter\": true,\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"encryptionMaterial\": {\n" - + " \"queryStageMasterKey\": \"EXAMPLE_QUERY_STAGE_MASTER_KEY\",\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"smkId\": 123\n" - + " },\n" - + " \"stageInfo\": {\n" - + " \"locationType\": \"S3\",\n" - + " \"location\": \"stage/location/foo/\",\n" - + " \"path\": \"tables/19805757505/\",\n" - + " \"region\": \"us-west-2\",\n" - + " \"storageAccount\": null,\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {\n" - + " \"AWS_KEY_ID\": \"EXAMPLE_AWS_KEY_ID\",\n" - + " \"AWS_SECRET_KEY\": \"EXAMPLE_AWS_SECRET_KEY\",\n" - + " \"AWS_TOKEN\": \"EXAMPLE_AWS_TOKEN\",\n" - + " \"AWS_ID\": \"EXAMPLE_AWS_ID\",\n" - + " \"AWS_KEY\": \"EXAMPLE_AWS_KEY\"\n" - + " },\n" - + " \"presignedUrl\": null,\n" - + " \"endPoint\": \"s3-fips.us-east-1.amazonaws.com\"\n" - + " },\n" - + " \"command\": \"UPLOAD\",\n" - + " \"kind\": null,\n" - + " \"operation\": \"Node\"\n" - + " },\n" - + " \"code\": null,\n" - + " \"message\": null,\n" - + " \"success\": true\n" - + "}"; - - private final String exampleS3JsonString = - "{\n" - + " \"data\": {\n" - + " \"uploadInfo\": {\n" - + " \"locationType\": \"S3\",\n" - + " \"location\": \"example/location\",\n" - + " \"path\": \"tables/19805757505/\",\n" - + " \"region\": \"us-west-2\",\n" - + " \"storageAccount\": null,\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {\n" - + " \"AWS_KEY_ID\": \"EXAMPLE_AWS_KEY_ID\",\n" - + " \"AWS_SECRET_KEY\": \"EXAMPLE_AWS_SECRET_KEY\",\n" - + " \"AWS_TOKEN\": \"EXAMPLE_AWS_TOKEN\",\n" - + " \"AWS_ID\": \"EXAMPLE_AWS_ID\",\n" - + " \"AWS_KEY\": \"EXAMPLE_AWS_KEY\"\n" - + " },\n" - + " \"presignedUrl\": null,\n" - + " \"endPoint\": null\n" - + " },\n" - + " \"src_locations\": [\n" - + " \"/tmp/files/orders_100.csv\"\n" - + " ],\n" - + " \"parallel\": 4,\n" - + " \"threshold\": 209715200,\n" - + " \"autoCompress\": true,\n" - + " \"overwrite\": false,\n" - + " \"sourceCompression\": \"auto_detect\",\n" - + " \"clientShowEncryptionParameter\": true,\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"encryptionMaterial\": {\n" - + " \"queryStageMasterKey\": \"EXAMPLE_QUERY_STAGE_MASTER_KEY\",\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"smkId\": 123\n" - + " },\n" - + " \"stageInfo\": {\n" - + " \"locationType\": \"S3\",\n" - + " \"location\": \"stage/location/foo/\",\n" - + " \"path\": \"tables/19805757505/\",\n" - + " \"region\": \"us-west-2\",\n" - + " \"storageAccount\": null,\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"useS3RegionalUrl\": true,\n" - + " \"creds\": {\n" - + " \"AWS_KEY_ID\": \"EXAMPLE_AWS_KEY_ID\",\n" - + " \"AWS_SECRET_KEY\": \"EXAMPLE_AWS_SECRET_KEY\",\n" - + " \"AWS_TOKEN\": \"EXAMPLE_AWS_TOKEN\",\n" - + " \"AWS_ID\": \"EXAMPLE_AWS_ID\",\n" - + " \"AWS_KEY\": \"EXAMPLE_AWS_KEY\"\n" - + " },\n" - + " \"presignedUrl\": null,\n" - + " \"endPoint\": null\n" - + " },\n" - + " \"command\": \"UPLOAD\",\n" - + " \"kind\": null,\n" - + " \"operation\": \"Node\"\n" - + " },\n" - + " \"code\": null,\n" - + " \"message\": null,\n" - + " \"success\": true\n" - + "}"; - - private final String exampleAzureJsonString = - "{\n" - + " \"data\": {\n" - + " \"uploadInfo\": {\n" - + " \"locationType\": \"AZURE\",\n" - + " \"location\": \"EXAMPLE_LOCATION/\",\n" - + " \"path\": \"EXAMPLE_PATH/\",\n" - + " \"region\": \"westus\",\n" - + " \"storageAccount\": \"sfcdev2stage\",\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {\n" - + " \"AZURE_SAS_TOKEN\": \"EXAMPLE_AZURE_SAS_TOKEN\"\n" - + " },\n" - + " \"presignedUrl\": null,\n" - + " \"endPoint\": \"blob.core.windows.net\"\n" - + " },\n" - + " \"src_locations\": [\n" - + " \"/foo/orders_100.csv\"\n" - + " ],\n" - + " \"parallel\": 4,\n" - + " \"threshold\": 209715200,\n" - + " \"autoCompress\": true,\n" - + " \"overwrite\": false,\n" - + " \"sourceCompression\": \"auto_detect\",\n" - + " \"clientShowEncryptionParameter\": false,\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"encryptionMaterial\": {\n" - + " \"queryStageMasterKey\": \"EXAMPLE_QUERY_STAGE_MASTER_KEY\",\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"smkId\": 123\n" - + " },\n" - + " \"stageInfo\": {\n" - + " \"locationType\": \"AZURE\",\n" - + " \"location\": \"EXAMPLE_LOCATION/\",\n" - + " \"path\": \"EXAMPLE_PATH/\",\n" - + " \"region\": \"westus\",\n" - + " \"storageAccount\": \"EXAMPLE_STORAGE_ACCOUNT\",\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {\n" - + " \"AZURE_SAS_TOKEN\": \"EXAMPLE_AZURE_SAS_TOKEN\"\n" - + " },\n" - + " \"presignedUrl\": null,\n" - + " \"endPoint\": \"blob.core.windows.net\"\n" - + " },\n" - + " \"command\": \"UPLOAD\",\n" - + " \"kind\": null,\n" - + " \"operation\": \"Node\"\n" - + " },\n" - + " \"code\": null,\n" - + " \"message\": null,\n" - + " \"success\": true\n" - + "}"; - - private final String exampleGCSJsonString = - "{\n" - + " \"data\": {\n" - + " \"uploadInfo\": {\n" - + " \"locationType\": \"GCS\",\n" - + " \"location\": \"foo/tables/9224/\",\n" - + " \"path\": \"tables/9224/\",\n" - + " \"region\": \"US-WEST1\",\n" - + " \"storageAccount\": \"\",\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {},\n" - + " \"presignedUrl\": \"EXAMPLE_PRESIGNED_URL\",\n" - + " \"endPoint\": \"\"\n" - + " },\n" - + " \"src_locations\": [\n" - + " \"/foo/bart/orders_100.csv\"\n" - + " ],\n" - + " \"parallel\": 4,\n" - + " \"threshold\": 209715200,\n" - + " \"autoCompress\": true,\n" - + " \"overwrite\": false,\n" - + " \"sourceCompression\": \"auto_detect\",\n" - + " \"clientShowEncryptionParameter\": false,\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"encryptionMaterial\": {\n" - + " \"queryStageMasterKey\": \"EXAMPLE_QUERY_STAGE_MASTER_KEY\",\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"smkId\": 123\n" - + " },\n" - + " \"stageInfo\": {\n" - + " \"locationType\": \"GCS\",\n" - + " \"location\": \"foo/tables/9224/\",\n" - + " \"path\": \"tables/9224/\",\n" - + " \"region\": \"US-WEST1\",\n" - + " \"storageAccount\": \"\",\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {},\n" - + " \"presignedUrl\": \"EXAMPLE_PRESIGNED_URL\",\n" - + " \"endPoint\": \"\"\n" - + " },\n" - + " \"command\": \"UPLOAD\",\n" - + " \"kind\": null,\n" - + " \"operation\": \"Node\"\n" - + " },\n" - + " \"code\": null,\n" - + " \"message\": null,\n" - + " \"success\": true\n" - + "}"; - private final String exampleGCSJsonStringWithUseRegionalUrl = - "{\n" - + " \"data\": {\n" - + " \"uploadInfo\": {\n" - + " \"locationType\": \"GCS\",\n" - + " \"useRegionalUrl\": true,\n" - + " \"location\": \"foo/tables/9224/\",\n" - + " \"path\": \"tables/9224/\",\n" - + " \"region\": \"US-WEST1\",\n" - + " \"storageAccount\": \"\",\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {},\n" - + " \"presignedUrl\": \"EXAMPLE_PRESIGNED_URL\",\n" - + " \"endPoint\": \"\"\n" - + " },\n" - + " \"src_locations\": [\n" - + " \"/foo/bart/orders_100.csv\"\n" - + " ],\n" - + " \"parallel\": 4,\n" - + " \"threshold\": 209715200,\n" - + " \"autoCompress\": true,\n" - + " \"overwrite\": false,\n" - + " \"sourceCompression\": \"auto_detect\",\n" - + " \"clientShowEncryptionParameter\": false,\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"encryptionMaterial\": {\n" - + " \"queryStageMasterKey\": \"EXAMPLE_QUERY_STAGE_MASTER_KEY\",\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"smkId\": 123\n" - + " },\n" - + " \"stageInfo\": {\n" - + " \"locationType\": \"GCS\",\n" - + " \"useRegionalUrl\": true,\n" - + " \"location\": \"foo/tables/9224/\",\n" - + " \"path\": \"tables/9224/\",\n" - + " \"region\": \"US-WEST1\",\n" - + " \"storageAccount\": \"\",\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {},\n" - + " \"presignedUrl\": \"EXAMPLE_PRESIGNED_URL\",\n" - + " \"endPoint\": \"\"\n" - + " },\n" - + " \"command\": \"UPLOAD\",\n" - + " \"kind\": null,\n" - + " \"operation\": \"Node\"\n" - + " },\n" - + " \"code\": null,\n" - + " \"message\": null,\n" - + " \"success\": true\n" - + "}"; - private final String exampleGCSJsonStringWithEndpoint = - "{\n" - + " \"data\": {\n" - + " \"uploadInfo\": {\n" - + " \"locationType\": \"GCS\",\n" - + " \"location\": \"foo/tables/9224/\",\n" - + " \"path\": \"tables/9224/\",\n" - + " \"region\": \"US-WEST1\",\n" - + " \"storageAccount\": \"\",\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {},\n" - + " \"presignedUrl\": \"EXAMPLE_PRESIGNED_URL\",\n" - + " \"endPoint\": \"example.com\"\n" - + " },\n" - + " \"src_locations\": [\n" - + " \"/foo/bart/orders_100.csv\"\n" - + " ],\n" - + " \"parallel\": 4,\n" - + " \"threshold\": 209715200,\n" - + " \"autoCompress\": true,\n" - + " \"overwrite\": false,\n" - + " \"sourceCompression\": \"auto_detect\",\n" - + " \"clientShowEncryptionParameter\": false,\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"encryptionMaterial\": {\n" - + " \"queryStageMasterKey\": \"EXAMPLE_QUERY_STAGE_MASTER_KEY\",\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"smkId\": 123\n" - + " },\n" - + " \"stageInfo\": {\n" - + " \"locationType\": \"GCS\",\n" - + " \"location\": \"foo/tables/9224/\",\n" - + " \"path\": \"tables/9224/\",\n" - + " \"region\": \"US-WEST1\",\n" - + " \"storageAccount\": \"\",\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {},\n" - + " \"presignedUrl\": \"EXAMPLE_PRESIGNED_URL\",\n" - + " \"endPoint\": \"example.com\"\n" - + " },\n" - + " \"command\": \"UPLOAD\",\n" - + " \"kind\": null,\n" - + " \"operation\": \"Node\"\n" - + " },\n" - + " \"code\": null,\n" - + " \"message\": null,\n" - + " \"success\": true\n" - + "}"; - - protected JsonNode exampleS3JsonNode; - protected JsonNode exampleS3StageEndpointJsonNode; - protected JsonNode exampleAzureJsonNode; - protected JsonNode exampleGCSJsonNode; - protected JsonNode exampleGCSJsonNodeWithUseRegionalUrl; - protected JsonNode exampleGCSJsonNodeWithEndPoint; - protected List exampleNodes; + private static final ObjectMapper mapper = new ObjectMapper(); + + static JsonNode exampleS3JsonNode; + static JsonNode exampleS3StageEndpointJsonNode; + static JsonNode exampleAzureJsonNode; + static JsonNode exampleGCSJsonNode; + static JsonNode exampleGCSJsonNodeWithUseRegionalUrl; + static JsonNode exampleGCSJsonNodeWithEndPoint; + static List exampleNodes; + + private static JsonNode readJsonFromFile(String name) throws IOException { + try (InputStream is = + FileUploaderPrep.class.getResourceAsStream("/FileUploaderPrep/" + name + ".json")) { + return mapper.readTree(is); + } + } - @Before - public void setup() throws Exception { - exampleS3JsonNode = mapper.readTree(exampleS3JsonString); - exampleS3StageEndpointJsonNode = mapper.readTree(exampleS3JsonStringWithStageEndpoint); - exampleAzureJsonNode = mapper.readTree(exampleAzureJsonString); - exampleGCSJsonNode = mapper.readTree(exampleGCSJsonString); - exampleGCSJsonNodeWithUseRegionalUrl = mapper.readTree(exampleGCSJsonStringWithUseRegionalUrl); - exampleGCSJsonNodeWithEndPoint = mapper.readTree(exampleGCSJsonStringWithEndpoint); - exampleNodes = Arrays.asList(exampleS3JsonNode, exampleAzureJsonNode, exampleGCSJsonNode); + @BeforeClass + public static void setup() throws Exception { + exampleS3JsonNode = readJsonFromFile("exampleS3"); + exampleS3StageEndpointJsonNode = readJsonFromFile("exampleS3WithStageEndpoint"); + exampleAzureJsonNode = readJsonFromFile("exampleAzure"); + exampleGCSJsonNode = readJsonFromFile("exampleGCS"); + exampleGCSJsonNodeWithUseRegionalUrl = readJsonFromFile("exampleGCSWithUseRegionalUrl"); + exampleGCSJsonNodeWithEndPoint = readJsonFromFile("exampleGCSWithEndpoint"); + exampleNodes = + Arrays.asList( + exampleS3JsonNode, + exampleAzureJsonNode, + exampleGCSJsonNode, + exampleGCSJsonNodeWithUseRegionalUrl, + exampleGCSJsonNodeWithEndPoint); } } diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderSessionlessTest.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderSessionlessTest.java index 94fd734f2..f5fb7f719 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderSessionlessTest.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderSessionlessTest.java @@ -6,6 +6,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -320,7 +321,7 @@ public void testGetFileTransferMetadatasUploadError() throws Exception { JsonNode downloadNode = mapper.readTree("{\"data\": {\"command\": \"DOWNLOAD\"}}"); try { SnowflakeFileTransferAgent.getFileTransferMetadatas(downloadNode); - assertTrue(false); + fail(); } catch (SnowflakeSQLException err) { Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode()); Assert.assertEquals( @@ -333,7 +334,7 @@ public void testGetFileTransferMetadatasEncryptionMaterialError() throws Excepti JsonNode garbageNode = mapper.readTree("{\"data\": {\"src_locations\": [1, 2]}}"); try { SnowflakeFileTransferAgent.getFileTransferMetadatas(garbageNode); - assertTrue(false); + fail(); } catch (SnowflakeSQLException err) { Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode()); assertTrue( @@ -348,7 +349,7 @@ public void testGetFileTransferMetadatasUnsupportedLocationError() throws Except foo.put("locationType", "LOCAL_FS"); try { SnowflakeFileTransferAgent.getFileTransferMetadatas(modifiedNode); - assertTrue(false); + fail(); } catch (SnowflakeSQLException err) { Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode()); assertTrue(err.getMessage().contains("JDBC driver internal error: This API only supports")); @@ -360,7 +361,7 @@ public void testGetFileTransferMetadatasSrcLocationsArrayError() throws JsonProc JsonNode garbageNode = mapper.readTree("{\"data\": {\"src_locations\": \"abc\"}}"); try { SnowflakeFileTransferAgent.getFileTransferMetadatas(garbageNode); - assertTrue(false); + fail(); } catch (SnowflakeSQLException err) { Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode()); assertTrue( @@ -375,7 +376,7 @@ public void testGetFileMetadatasEncryptionMaterialsException() { foo.put("encryptionMaterial", "[1, 2, 3]]"); try { SnowflakeFileTransferAgent.getFileTransferMetadatas(modifiedNode); - assertTrue(false); + fail(); } catch (SnowflakeSQLException err) { Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode()); assertTrue(err.getMessage().contains("Failed to parse encryptionMaterial")); diff --git a/src/test/resources/FileUploaderPrep/exampleAzure.json b/src/test/resources/FileUploaderPrep/exampleAzure.json new file mode 100644 index 000000000..a2b1835c3 --- /dev/null +++ b/src/test/resources/FileUploaderPrep/exampleAzure.json @@ -0,0 +1,51 @@ +{ + "data": { + "uploadInfo": { + "locationType": "AZURE", + "location": "EXAMPLE_LOCATION/", + "path": "EXAMPLE_PATH/", + "region": "westus", + "storageAccount": "sfcdev2stage", + "isClientSideEncrypted": true, + "creds": { + "AZURE_SAS_TOKEN": "EXAMPLE_AZURE_SAS_TOKEN" + }, + "presignedUrl": null, + "endPoint": "blob.core.windows.net" + }, + "src_locations": [ + "/foo/orders_100.csv" + ], + "parallel": 4, + "threshold": 209715200, + "autoCompress": true, + "overwrite": false, + "sourceCompression": "auto_detect", + "clientShowEncryptionParameter": false, + "queryId": "EXAMPLE_QUERY_ID", + "encryptionMaterial": { + "queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY", + "queryId": "EXAMPLE_QUERY_ID", + "smkId": 123 + }, + "stageInfo": { + "locationType": "AZURE", + "location": "EXAMPLE_LOCATION/", + "path": "EXAMPLE_PATH/", + "region": "westus", + "storageAccount": "EXAMPLE_STORAGE_ACCOUNT", + "isClientSideEncrypted": true, + "creds": { + "AZURE_SAS_TOKEN": "EXAMPLE_AZURE_SAS_TOKEN" + }, + "presignedUrl": null, + "endPoint": "blob.core.windows.net" + }, + "command": "UPLOAD", + "kind": null, + "operation": "Node" + }, + "code": null, + "message": null, + "success": true +} \ No newline at end of file diff --git a/src/test/resources/FileUploaderPrep/exampleGCS.json b/src/test/resources/FileUploaderPrep/exampleGCS.json new file mode 100644 index 000000000..8cd605f1c --- /dev/null +++ b/src/test/resources/FileUploaderPrep/exampleGCS.json @@ -0,0 +1,47 @@ +{ + "data": { + "uploadInfo": { + "locationType": "GCS", + "location": "foo/tables/9224/", + "path": "tables/9224/", + "region": "US-WEST1", + "storageAccount": "", + "isClientSideEncrypted": true, + "creds": {}, + "presignedUrl": "EXAMPLE_PRESIGNED_URL", + "endPoint": "" + }, + "src_locations": [ + "/foo/bart/orders_100.csv" + ], + "parallel": 4, + "threshold": 209715200, + "autoCompress": true, + "overwrite": false, + "sourceCompression": "auto_detect", + "clientShowEncryptionParameter": false, + "queryId": "EXAMPLE_QUERY_ID", + "encryptionMaterial": { + "queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY", + "queryId": "EXAMPLE_QUERY_ID", + "smkId": 123 + }, + "stageInfo": { + "locationType": "GCS", + "location": "foo/tables/9224/", + "path": "tables/9224/", + "region": "US-WEST1", + "storageAccount": "", + "isClientSideEncrypted": true, + "creds": {}, + "presignedUrl": "EXAMPLE_PRESIGNED_URL", + "endPoint": "" + }, + "command": "UPLOAD", + "kind": null, + "operation": "Node" + }, + "code": null, + "message": null, + "success": true +} \ No newline at end of file diff --git a/src/test/resources/FileUploaderPrep/exampleGCSWithEndpoint.json b/src/test/resources/FileUploaderPrep/exampleGCSWithEndpoint.json new file mode 100644 index 000000000..8ba946c76 --- /dev/null +++ b/src/test/resources/FileUploaderPrep/exampleGCSWithEndpoint.json @@ -0,0 +1,47 @@ +{ + "data": { + "uploadInfo": { + "locationType": "GCS", + "location": "foo/tables/9224/", + "path": "tables/9224/", + "region": "US-WEST1", + "storageAccount": "", + "isClientSideEncrypted": true, + "creds": {}, + "presignedUrl": "EXAMPLE_PRESIGNED_URL", + "endPoint": "example.com" + }, + "src_locations": [ + "/foo/bart/orders_100.csv" + ], + "parallel": 4, + "threshold": 209715200, + "autoCompress": true, + "overwrite": false, + "sourceCompression": "auto_detect", + "clientShowEncryptionParameter": false, + "queryId": "EXAMPLE_QUERY_ID", + "encryptionMaterial": { + "queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY", + "queryId": "EXAMPLE_QUERY_ID", + "smkId": 123 + }, + "stageInfo": { + "locationType": "GCS", + "location": "foo/tables/9224/", + "path": "tables/9224/", + "region": "US-WEST1", + "storageAccount": "", + "isClientSideEncrypted": true, + "creds": {}, + "presignedUrl": "EXAMPLE_PRESIGNED_URL", + "endPoint": "example.com" + }, + "command": "UPLOAD", + "kind": null, + "operation": "Node" + }, + "code": null, + "message": null, + "success": true +} \ No newline at end of file diff --git a/src/test/resources/FileUploaderPrep/exampleGCSWithUseRegionalUrl.json b/src/test/resources/FileUploaderPrep/exampleGCSWithUseRegionalUrl.json new file mode 100644 index 000000000..79f4dc678 --- /dev/null +++ b/src/test/resources/FileUploaderPrep/exampleGCSWithUseRegionalUrl.json @@ -0,0 +1,49 @@ +{ + "data": { + "uploadInfo": { + "locationType": "GCS", + "useRegionalUrl": true, + "location": "foo/tables/9224/", + "path": "tables/9224/", + "region": "US-WEST1", + "storageAccount": "", + "isClientSideEncrypted": true, + "creds": {}, + "presignedUrl": "EXAMPLE_PRESIGNED_URL", + "endPoint": "" + }, + "src_locations": [ + "/foo/bart/orders_100.csv" + ], + "parallel": 4, + "threshold": 209715200, + "autoCompress": true, + "overwrite": false, + "sourceCompression": "auto_detect", + "clientShowEncryptionParameter": false, + "queryId": "EXAMPLE_QUERY_ID", + "encryptionMaterial": { + "queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY", + "queryId": "EXAMPLE_QUERY_ID", + "smkId": 123 + }, + "stageInfo": { + "locationType": "GCS", + "useRegionalUrl": true, + "location": "foo/tables/9224/", + "path": "tables/9224/", + "region": "US-WEST1", + "storageAccount": "", + "isClientSideEncrypted": true, + "creds": {}, + "presignedUrl": "EXAMPLE_PRESIGNED_URL", + "endPoint": "" + }, + "command": "UPLOAD", + "kind": null, + "operation": "Node" + }, + "code": null, + "message": null, + "success": true +} \ No newline at end of file diff --git a/src/test/resources/FileUploaderPrep/exampleS3.json b/src/test/resources/FileUploaderPrep/exampleS3.json new file mode 100644 index 000000000..eadc166d8 --- /dev/null +++ b/src/test/resources/FileUploaderPrep/exampleS3.json @@ -0,0 +1,60 @@ +{ + "data": { + "uploadInfo": { + "locationType": "S3", + "location": "example/location", + "path": "tables/19805757505/", + "region": "us-west-2", + "storageAccount": null, + "isClientSideEncrypted": true, + "creds": { + "AWS_KEY_ID": "EXAMPLE_AWS_KEY_ID", + "AWS_SECRET_KEY": "EXAMPLE_AWS_SECRET_KEY", + "AWS_TOKEN": "EXAMPLE_AWS_TOKEN", + "AWS_ID": "EXAMPLE_AWS_ID", + "AWS_KEY": "EXAMPLE_AWS_KEY" + }, + "presignedUrl": null, + "endPoint": null + }, + "src_locations": [ + "/tmp/files/orders_100.csv" + ], + "parallel": 4, + "threshold": 209715200, + "autoCompress": true, + "overwrite": false, + "sourceCompression": "auto_detect", + "clientShowEncryptionParameter": true, + "queryId": "EXAMPLE_QUERY_ID", + "encryptionMaterial": { + "queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY", + "queryId": "EXAMPLE_QUERY_ID", + "smkId": 123 + }, + "stageInfo": { + "locationType": "S3", + "location": "stage/location/foo/", + "path": "tables/19805757505/", + "region": "us-west-2", + "storageAccount": null, + "isClientSideEncrypted": true, + "useS3RegionalUrl": true, + "creds": { + "AWS_KEY_ID": "EXAMPLE_AWS_KEY_ID", + "AWS_SECRET_KEY": "EXAMPLE_AWS_SECRET_KEY", + "AWS_TOKEN": "EXAMPLE_AWS_TOKEN", + "AWS_ID": "EXAMPLE_AWS_ID", + "AWS_KEY": "EXAMPLE_AWS_KEY" + }, + "presignedUrl": null, + "endPoint": null + }, + "command": "UPLOAD", + "kind": null, + "operation": "Node" + }, + "code": null, + "message": null, + "success": true +} \ No newline at end of file diff --git a/src/test/resources/FileUploaderPrep/exampleS3WithStageEndpoint.json b/src/test/resources/FileUploaderPrep/exampleS3WithStageEndpoint.json new file mode 100644 index 000000000..32b8a66a1 --- /dev/null +++ b/src/test/resources/FileUploaderPrep/exampleS3WithStageEndpoint.json @@ -0,0 +1,59 @@ +{ + "data": { + "uploadInfo": { + "locationType": "S3", + "location": "example/location", + "path": "tables/19805757505/", + "region": "us-west-2", + "storageAccount": null, + "isClientSideEncrypted": true, + "creds": { + "AWS_KEY_ID": "EXAMPLE_AWS_KEY_ID", + "AWS_SECRET_KEY": "EXAMPLE_AWS_SECRET_KEY", + "AWS_TOKEN": "EXAMPLE_AWS_TOKEN", + "AWS_ID": "EXAMPLE_AWS_ID", + "AWS_KEY": "EXAMPLE_AWS_KEY" + }, + "presignedUrl": null, + "endPoint": null + }, + "src_locations": [ + "/tmp/files/orders_100.csv" + ], + "parallel": 4, + "threshold": 209715200, + "autoCompress": true, + "overwrite": false, + "sourceCompression": "auto_detect", + "clientShowEncryptionParameter": true, + "queryId": "EXAMPLE_QUERY_ID", + "encryptionMaterial": { + "queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY", + "queryId": "EXAMPLE_QUERY_ID", + "smkId": 123 + }, + "stageInfo": { + "locationType": "S3", + "location": "stage/location/foo/", + "path": "tables/19805757505/", + "region": "us-west-2", + "storageAccount": null, + "isClientSideEncrypted": true, + "creds": { + "AWS_KEY_ID": "EXAMPLE_AWS_KEY_ID", + "AWS_SECRET_KEY": "EXAMPLE_AWS_SECRET_KEY", + "AWS_TOKEN": "EXAMPLE_AWS_TOKEN", + "AWS_ID": "EXAMPLE_AWS_ID", + "AWS_KEY": "EXAMPLE_AWS_KEY" + }, + "presignedUrl": null, + "endPoint": "s3-fips.us-east-1.amazonaws.com" + }, + "command": "UPLOAD", + "kind": null, + "operation": "Node" + }, + "code": null, + "message": null, + "success": true +} \ No newline at end of file