diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 2607c5d46..ef331f720 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -29,7 +29,7 @@ jobs: name: Build runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Build shell: bash env: @@ -53,7 +53,7 @@ jobs: java-version: ${{ matrix.runConfig.javaVersion }} distribution: 'temurin' cache: maven - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.7' architecture: 'x64' @@ -83,7 +83,7 @@ jobs: java-version: ${{ matrix.runConfig.javaVersion }} distribution: 'temurin' cache: maven - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.7' - name: Install Homebrew Bash @@ -110,7 +110,7 @@ jobs: category: ['TestCategoryResultSet,TestCategoryStatement,TestCategoryLoader', 'TestCategoryOthers', 'TestCategoryArrow,TestCategoryConnection,TestCategoryCore,TestCategoryDiagnostic', 'TestCategoryFips'] additionalMavenProfile: ['', '-Dthin-jar'] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Tests shell: bash env: @@ -132,7 +132,7 @@ jobs: category: ['TestCategoryOthers', 'TestCategoryConnection,TestCategoryStatement', 'TestCategoryCore,TestCategoryLoader,TestCategoryResultSet'] is_old_driver: ['true'] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Tests shell: bash env: diff --git a/.github/workflows/check-style.yml b/.github/workflows/check-style.yml index 221651298..d26f41865 100644 --- a/.github/workflows/check-style.yml +++ b/.github/workflows/check-style.yml @@ -9,7 +9,7 @@ jobs: name: Check Style runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Check Style shell: bash run: mvn clean validate --batch-mode --show-version -P check-style diff --git a/.github/workflows/jira_close.yml b/.github/workflows/jira_close.yml index dfcb8bc73..0dacf7fab 100644 --- a/.github/workflows/jira_close.yml +++ b/.github/workflows/jira_close.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: snowflakedb/gh-actions ref: jira_v1 diff --git a/.github/workflows/jira_issue.yml b/.github/workflows/jira_issue.yml index 943ad70aa..92501da8f 100644 --- a/.github/workflows/jira_issue.yml +++ b/.github/workflows/jira_issue.yml @@ -14,7 +14,7 @@ jobs: if: ((github.event_name == 'issue_comment' && github.event.comment.body == 'recreate jira' && github.event.comment.user.login == 'sfc-gh-mkeller') || (github.event_name == 'issues' && github.event.pull_request.user.login != 'whitesource-for-github-com[bot]')) steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: repository: snowflakedb/gh-actions ref: jira_v1 diff --git a/.github/workflows/snyk-issue.yml b/.github/workflows/snyk-issue.yml index 7b58bb12a..1e36dae35 100644 --- a/.github/workflows/snyk-issue.yml +++ b/.github/workflows/snyk-issue.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: checkout action - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: snowflakedb/whitesource-actions token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }} diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml index 5fc21951b..0c101e391 100644 --- a/.github/workflows/snyk-pr.yml +++ b/.github/workflows/snyk-pr.yml @@ -15,13 +15,13 @@ jobs: if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.ref }} fetch-depth: 0 - name: checkout action - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: snowflakedb/whitesource-actions token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }} diff --git a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java index 2b660eb08..4213b33b0 100644 --- a/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java +++ b/src/main/java/net/snowflake/client/jdbc/SnowflakeFileTransferAgent.java @@ -1111,8 +1111,16 @@ static StageInfo getStageInfo(JsonNode jsonNode, SFSession session) throws Snowf // specifically // for FIPS or VPCE S3 endpoint. SNOW-652696 String endPoint = null; - if ("AZURE".equalsIgnoreCase(stageLocationType) || "S3".equalsIgnoreCase(stageLocationType)) { + if ("AZURE".equalsIgnoreCase(stageLocationType) + || "S3".equalsIgnoreCase(stageLocationType) + || "GCS".equalsIgnoreCase(stageLocationType)) { endPoint = jsonNode.path("data").path("stageInfo").findValue("endPoint").asText(); + if ("GCS".equalsIgnoreCase(stageLocationType) + && endPoint != null + && (endPoint.trim().isEmpty() || "null".equals(endPoint))) { + // setting to null to preserve previous behaviour for GCS + endPoint = null; + } } String stgAcct = null; @@ -1179,6 +1187,8 @@ static StageInfo getStageInfo(JsonNode jsonNode, SFSession session) throws Snowf } } + setupUseRegionalUrl(jsonNode, stageInfo); + if (stageInfo.getStageType() == StageInfo.StageType.S3) { if (session == null) { // This node's value is set if PUT is used without Session. (For Snowpipe Streaming, we rely @@ -1200,6 +1210,18 @@ static StageInfo getStageInfo(JsonNode jsonNode, SFSession session) throws Snowf return stageInfo; } + private static void setupUseRegionalUrl(JsonNode jsonNode, StageInfo stageInfo) { + if (stageInfo.getStageType() != StageInfo.StageType.GCS + && stageInfo.getStageType() != StageInfo.StageType.S3) { + return; + } + JsonNode useRegionalURLNode = jsonNode.path("data").path("stageInfo").path("useRegionalUrl"); + if (!useRegionalURLNode.isMissingNode()) { + boolean useRegionalURL = useRegionalURLNode.asBoolean(false); + stageInfo.setUseRegionalUrl(useRegionalURL); + } + } + /** * A helper method to verify if the local file path from GS matches what's parsed locally. This is * for security purpose as documented in SNOW-15153. diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java index 188ba40d4..d6bf6ba84 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/SnowflakeGCSClient.java @@ -18,9 +18,11 @@ import com.google.api.gax.rpc.FixedHeaderProvider; import com.google.auth.oauth2.AccessToken; import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.NoCredentials; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.HttpStorageOptions; import com.google.cloud.storage.Storage; import com.google.cloud.storage.Storage.BlobListOption; import com.google.cloud.storage.StorageException; @@ -1312,6 +1314,8 @@ private void setupGCSClient( if (accessToken != null) { // We are authenticated with an oauth access token. StorageOptions.Builder builder = StorageOptions.newBuilder(); + stage.gcsCustomEndpoint().ifPresent(builder::setHost); + if (areDisabledGcsDefaultCredentials(session)) { logger.debug( "Adding explicit credentials to avoid default credential lookup by the GCS client"); @@ -1329,7 +1333,10 @@ private void setupGCSClient( .getService(); } else { // Use anonymous authentication. - this.gcsClient = StorageOptions.getUnauthenticatedInstance().getService(); + HttpStorageOptions.Builder builder = + HttpStorageOptions.newBuilder().setCredentials(NoCredentials.getInstance()); + stage.gcsCustomEndpoint().ifPresent(builder::setHost); + this.gcsClient = builder.build().getService(); } if (encMat != null) { diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StageInfo.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StageInfo.java index 7a8bf4d36..3a14b8fa0 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StageInfo.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StageInfo.java @@ -2,10 +2,17 @@ import java.io.Serializable; import java.util.Map; +import java.util.Optional; import java.util.Properties; +import net.snowflake.client.core.SnowflakeJdbcInternalApi; -/** Encapsulates all the required stage properties used by GET/PUT for Azure and S3 stages */ +/** Encapsulates all the required stage properties used by GET/PUT for Azure, GCS and S3 stages */ public class StageInfo implements Serializable { + + // me-central2 GCS region always use regional urls + // TODO SNOW-1818804: the value is hardcoded now, but it should be server driven + private static final String GCS_REGION_ME_CENTRAL_2 = "me-central2"; + public enum StageType { S3, AZURE, @@ -17,12 +24,18 @@ public enum StageType { private StageType stageType; // The stage type private String location; // The container or bucket private Map credentials; // the credentials required for the stage - private String region; // AWS/S3/GCS region (S3/GCS only) - private String endPoint; // The Azure Storage endpoint (Azure only) + private String region; // S3/GCS region + // An endpoint (Azure, AWS FIPS and GCS custom endpoint override) + private String endPoint; private String storageAccount; // The Azure Storage account (Azure only) private String presignedUrl; // GCS gives us back a presigned URL instead of a cred private boolean isClientSideEncrypted; // whether to encrypt/decrypt files on the stage - private boolean useS3RegionalUrl; // whether to use s3 regional URL (AWS Only) + // whether to use s3 regional URL (AWS Only) + // TODO SNOW-1818804: this field will be deprecated when the server returns {@link + // #useRegionalUrl} + private boolean useS3RegionalUrl; + // whether to use regional URL (AWS and GCS only) + private boolean useRegionalUrl; private Properties proxyProperties; /* @@ -166,6 +179,16 @@ public boolean getUseS3RegionalUrl() { return useS3RegionalUrl; } + @SnowflakeJdbcInternalApi + public void setUseRegionalUrl(boolean useRegionalUrl) { + this.useRegionalUrl = useRegionalUrl; + } + + @SnowflakeJdbcInternalApi + public boolean getUseRegionalUrl() { + return useRegionalUrl; + } + private static boolean isSpecified(String arg) { return !(arg == null || arg.equalsIgnoreCase("")); } @@ -173,9 +196,22 @@ private static boolean isSpecified(String arg) { public void setProxyProperties(Properties proxyProperties) { this.proxyProperties = proxyProperties; } - ; public Properties getProxyProperties() { return proxyProperties; } + + @SnowflakeJdbcInternalApi + public Optional gcsCustomEndpoint() { + if (stageType != StageType.GCS) { + return Optional.empty(); + } + if (endPoint != null && !endPoint.trim().isEmpty() && !"null".equals(endPoint)) { + return Optional.of(endPoint); + } + if (GCS_REGION_ME_CENTRAL_2.equalsIgnoreCase(region) || useRegionalUrl) { + return Optional.of(String.format("storage.%s.rep.googleapis.com", region.toLowerCase())); + } + return Optional.empty(); + } } diff --git a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageClientFactory.java b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageClientFactory.java index a321b6ebd..69d56e195 100644 --- a/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageClientFactory.java +++ b/src/main/java/net/snowflake/client/jdbc/cloud/storage/StorageClientFactory.java @@ -59,8 +59,9 @@ public SnowflakeStorageClient createClient( switch (stage.getStageType()) { case S3: boolean useS3RegionalUrl = - (stage.getUseS3RegionalUrl() - || (session != null && session.getUseRegionalS3EndpointsForPresignedURL())); + stage.getUseS3RegionalUrl() + || stage.getUseRegionalUrl() + || session != null && session.getUseRegionalS3EndpointsForPresignedURL(); return createS3Client( stage.getCredentials(), parallel, diff --git a/src/test/java/net/snowflake/client/jdbc/ConnectionLatestIT.java b/src/test/java/net/snowflake/client/jdbc/ConnectionLatestIT.java index 4dbbcb021..30ff6728f 100644 --- a/src/test/java/net/snowflake/client/jdbc/ConnectionLatestIT.java +++ b/src/test/java/net/snowflake/client/jdbc/ConnectionLatestIT.java @@ -274,7 +274,6 @@ public void testAsyncQueryOpenAndCloseConnection() throws SQLException, IOException, InterruptedException { // open connection and run asynchronous query String queryID = null; - QueryStatusV2 statusV2 = null; try (Connection con = getConnection(); Statement statement = con.createStatement(); ResultSet rs1 = @@ -288,7 +287,7 @@ public void testAsyncQueryOpenAndCloseConnection() await() .atMost(Duration.ofSeconds(5)) .until(() -> sfrs.getStatusV2().getStatus(), not(equalTo(QueryStatus.NO_DATA))); - statusV2 = sfrs.getStatusV2(); + QueryStatusV2 statusV2 = sfrs.getStatusV2(); // Query should take 60 seconds so should be running assertEquals(QueryStatus.RUNNING, statusV2.getStatus()); assertEquals(QueryStatus.RUNNING.name(), statusV2.getName()); @@ -305,7 +304,7 @@ public void testAsyncQueryOpenAndCloseConnection() assertEquals(SqlState.INVALID_PARAMETER_VALUE, e.getSQLState()); } try (ResultSet rs = con.unwrap(SnowflakeConnection.class).createResultSet(queryID)) { - statusV2 = rs.unwrap(SnowflakeResultSet.class).getStatusV2(); + QueryStatusV2 statusV2 = rs.unwrap(SnowflakeResultSet.class).getStatusV2(); // Assert status of query is a success assertEquals(QueryStatus.SUCCESS, statusV2.getStatus()); assertEquals("No error reported", statusV2.getErrorMessage()); @@ -318,27 +317,16 @@ public void testAsyncQueryOpenAndCloseConnection() .unwrap(SnowflakeStatement.class) .executeAsyncQuery("select * from nonexistentTable")) { Thread.sleep(100); - statusV2 = rs1.unwrap(SnowflakeResultSet.class).getStatusV2(); - // when GS response is slow, allow up to 1 second of retries to get final query status SnowflakeResultSet sfrs1 = rs1.unwrap(SnowflakeResultSet.class); await() .atMost(Duration.ofSeconds(10)) - .until( - () -> { - QueryStatus qs = sfrs1.getStatusV2().getStatus(); - return !(qs == QueryStatus.NO_DATA || qs == QueryStatus.RUNNING); - }); - // If GS response is too slow to return data, do nothing to avoid flaky test failure. If - // response has returned, - // assert it is the error message that we are expecting. - if (statusV2.getStatus() != QueryStatus.NO_DATA) { - assertEquals(QueryStatus.FAILED_WITH_ERROR, statusV2.getStatus()); - assertEquals(2003, statusV2.getErrorCode()); - assertEquals( - "SQL compilation error:\n" - + "Object 'NONEXISTENTTABLE' does not exist or not authorized.", - statusV2.getErrorMessage()); - } + .until(() -> sfrs1.getStatusV2().getStatus() == QueryStatus.FAILED_WITH_ERROR); + statusV2 = sfrs1.getStatusV2(); + assertEquals(2003, statusV2.getErrorCode()); + assertEquals( + "SQL compilation error:\n" + + "Object 'NONEXISTENTTABLE' does not exist or not authorized.", + statusV2.getErrorMessage()); } } } diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderPrep.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderPrep.java index d8aa8143f..d801a00ac 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderPrep.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderPrep.java @@ -6,254 +6,49 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; import java.util.List; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TemporaryFolder; /** File uploader test prep reused by IT/connection tests and sessionless tests */ abstract class FileUploaderPrep extends BaseJDBCTest { @Rule public TemporaryFolder folder = new TemporaryFolder(); - private ObjectMapper mapper = new ObjectMapper(); - private final String exampleS3JsonStringWithStageEndpoint = - "{\n" - + " \"data\": {\n" - + " \"uploadInfo\": {\n" - + " \"locationType\": \"S3\",\n" - + " \"location\": \"example/location\",\n" - + " \"path\": \"tables/19805757505/\",\n" - + " \"region\": \"us-west-2\",\n" - + " \"storageAccount\": null,\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {\n" - + " \"AWS_KEY_ID\": \"EXAMPLE_AWS_KEY_ID\",\n" - + " \"AWS_SECRET_KEY\": \"EXAMPLE_AWS_SECRET_KEY\",\n" - + " \"AWS_TOKEN\": \"EXAMPLE_AWS_TOKEN\",\n" - + " \"AWS_ID\": \"EXAMPLE_AWS_ID\",\n" - + " \"AWS_KEY\": \"EXAMPLE_AWS_KEY\"\n" - + " },\n" - + " \"presignedUrl\": null,\n" - + " \"endPoint\": null\n" - + " },\n" - + " \"src_locations\": [\n" - + " \"/tmp/files/orders_100.csv\"\n" - + " ],\n" - + " \"parallel\": 4,\n" - + " \"threshold\": 209715200,\n" - + " \"autoCompress\": true,\n" - + " \"overwrite\": false,\n" - + " \"sourceCompression\": \"auto_detect\",\n" - + " \"clientShowEncryptionParameter\": true,\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"encryptionMaterial\": {\n" - + " \"queryStageMasterKey\": \"EXAMPLE_QUERY_STAGE_MASTER_KEY\",\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"smkId\": 123\n" - + " },\n" - + " \"stageInfo\": {\n" - + " \"locationType\": \"S3\",\n" - + " \"location\": \"stage/location/foo/\",\n" - + " \"path\": \"tables/19805757505/\",\n" - + " \"region\": \"us-west-2\",\n" - + " \"storageAccount\": null,\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {\n" - + " \"AWS_KEY_ID\": \"EXAMPLE_AWS_KEY_ID\",\n" - + " \"AWS_SECRET_KEY\": \"EXAMPLE_AWS_SECRET_KEY\",\n" - + " \"AWS_TOKEN\": \"EXAMPLE_AWS_TOKEN\",\n" - + " \"AWS_ID\": \"EXAMPLE_AWS_ID\",\n" - + " \"AWS_KEY\": \"EXAMPLE_AWS_KEY\"\n" - + " },\n" - + " \"presignedUrl\": null,\n" - + " \"endPoint\": \"s3-fips.us-east-1.amazonaws.com\"\n" - + " },\n" - + " \"command\": \"UPLOAD\",\n" - + " \"kind\": null,\n" - + " \"operation\": \"Node\"\n" - + " },\n" - + " \"code\": null,\n" - + " \"message\": null,\n" - + " \"success\": true\n" - + "}"; - - private final String exampleS3JsonString = - "{\n" - + " \"data\": {\n" - + " \"uploadInfo\": {\n" - + " \"locationType\": \"S3\",\n" - + " \"location\": \"example/location\",\n" - + " \"path\": \"tables/19805757505/\",\n" - + " \"region\": \"us-west-2\",\n" - + " \"storageAccount\": null,\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {\n" - + " \"AWS_KEY_ID\": \"EXAMPLE_AWS_KEY_ID\",\n" - + " \"AWS_SECRET_KEY\": \"EXAMPLE_AWS_SECRET_KEY\",\n" - + " \"AWS_TOKEN\": \"EXAMPLE_AWS_TOKEN\",\n" - + " \"AWS_ID\": \"EXAMPLE_AWS_ID\",\n" - + " \"AWS_KEY\": \"EXAMPLE_AWS_KEY\"\n" - + " },\n" - + " \"presignedUrl\": null,\n" - + " \"endPoint\": null\n" - + " },\n" - + " \"src_locations\": [\n" - + " \"/tmp/files/orders_100.csv\"\n" - + " ],\n" - + " \"parallel\": 4,\n" - + " \"threshold\": 209715200,\n" - + " \"autoCompress\": true,\n" - + " \"overwrite\": false,\n" - + " \"sourceCompression\": \"auto_detect\",\n" - + " \"clientShowEncryptionParameter\": true,\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"encryptionMaterial\": {\n" - + " \"queryStageMasterKey\": \"EXAMPLE_QUERY_STAGE_MASTER_KEY\",\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"smkId\": 123\n" - + " },\n" - + " \"stageInfo\": {\n" - + " \"locationType\": \"S3\",\n" - + " \"location\": \"stage/location/foo/\",\n" - + " \"path\": \"tables/19805757505/\",\n" - + " \"region\": \"us-west-2\",\n" - + " \"storageAccount\": null,\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"useS3RegionalUrl\": true,\n" - + " \"creds\": {\n" - + " \"AWS_KEY_ID\": \"EXAMPLE_AWS_KEY_ID\",\n" - + " \"AWS_SECRET_KEY\": \"EXAMPLE_AWS_SECRET_KEY\",\n" - + " \"AWS_TOKEN\": \"EXAMPLE_AWS_TOKEN\",\n" - + " \"AWS_ID\": \"EXAMPLE_AWS_ID\",\n" - + " \"AWS_KEY\": \"EXAMPLE_AWS_KEY\"\n" - + " },\n" - + " \"presignedUrl\": null,\n" - + " \"endPoint\": null\n" - + " },\n" - + " \"command\": \"UPLOAD\",\n" - + " \"kind\": null,\n" - + " \"operation\": \"Node\"\n" - + " },\n" - + " \"code\": null,\n" - + " \"message\": null,\n" - + " \"success\": true\n" - + "}"; - - private final String exampleAzureJsonString = - "{\n" - + " \"data\": {\n" - + " \"uploadInfo\": {\n" - + " \"locationType\": \"AZURE\",\n" - + " \"location\": \"EXAMPLE_LOCATION/\",\n" - + " \"path\": \"EXAMPLE_PATH/\",\n" - + " \"region\": \"westus\",\n" - + " \"storageAccount\": \"sfcdev2stage\",\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {\n" - + " \"AZURE_SAS_TOKEN\": \"EXAMPLE_AZURE_SAS_TOKEN\"\n" - + " },\n" - + " \"presignedUrl\": null,\n" - + " \"endPoint\": \"blob.core.windows.net\"\n" - + " },\n" - + " \"src_locations\": [\n" - + " \"/foo/orders_100.csv\"\n" - + " ],\n" - + " \"parallel\": 4,\n" - + " \"threshold\": 209715200,\n" - + " \"autoCompress\": true,\n" - + " \"overwrite\": false,\n" - + " \"sourceCompression\": \"auto_detect\",\n" - + " \"clientShowEncryptionParameter\": false,\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"encryptionMaterial\": {\n" - + " \"queryStageMasterKey\": \"EXAMPLE_QUERY_STAGE_MASTER_KEY\",\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"smkId\": 123\n" - + " },\n" - + " \"stageInfo\": {\n" - + " \"locationType\": \"AZURE\",\n" - + " \"location\": \"EXAMPLE_LOCATION/\",\n" - + " \"path\": \"EXAMPLE_PATH/\",\n" - + " \"region\": \"westus\",\n" - + " \"storageAccount\": \"EXAMPLE_STORAGE_ACCOUNT\",\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {\n" - + " \"AZURE_SAS_TOKEN\": \"EXAMPLE_AZURE_SAS_TOKEN\"\n" - + " },\n" - + " \"presignedUrl\": null,\n" - + " \"endPoint\": \"blob.core.windows.net\"\n" - + " },\n" - + " \"command\": \"UPLOAD\",\n" - + " \"kind\": null,\n" - + " \"operation\": \"Node\"\n" - + " },\n" - + " \"code\": null,\n" - + " \"message\": null,\n" - + " \"success\": true\n" - + "}"; - - private final String exampleGCSJsonString = - "{\n" - + " \"data\": {\n" - + " \"uploadInfo\": {\n" - + " \"locationType\": \"GCS\",\n" - + " \"location\": \"foo/tables/9224/\",\n" - + " \"path\": \"tables/9224/\",\n" - + " \"region\": \"US-WEST1\",\n" - + " \"storageAccount\": \"\",\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {},\n" - + " \"presignedUrl\": \"EXAMPLE_PRESIGNED_URL\",\n" - + " \"endPoint\": \"\"\n" - + " },\n" - + " \"src_locations\": [\n" - + " \"/foo/bart/orders_100.csv\"\n" - + " ],\n" - + " \"parallel\": 4,\n" - + " \"threshold\": 209715200,\n" - + " \"autoCompress\": true,\n" - + " \"overwrite\": false,\n" - + " \"sourceCompression\": \"auto_detect\",\n" - + " \"clientShowEncryptionParameter\": false,\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"encryptionMaterial\": {\n" - + " \"queryStageMasterKey\": \"EXAMPLE_QUERY_STAGE_MASTER_KEY\",\n" - + " \"queryId\": \"EXAMPLE_QUERY_ID\",\n" - + " \"smkId\": 123\n" - + " },\n" - + " \"stageInfo\": {\n" - + " \"locationType\": \"GCS\",\n" - + " \"location\": \"foo/tables/9224/\",\n" - + " \"path\": \"tables/9224/\",\n" - + " \"region\": \"US-WEST1\",\n" - + " \"storageAccount\": \"\",\n" - + " \"isClientSideEncrypted\": true,\n" - + " \"creds\": {},\n" - + " \"presignedUrl\": \"EXAMPLE_PRESIGNED_URL\",\n" - + " \"endPoint\": \"\"\n" - + " },\n" - + " \"command\": \"UPLOAD\",\n" - + " \"kind\": null,\n" - + " \"operation\": \"Node\"\n" - + " },\n" - + " \"code\": null,\n" - + " \"message\": null,\n" - + " \"success\": true\n" - + "}"; - - protected JsonNode exampleS3JsonNode; - protected JsonNode exampleS3StageEndpointJsonNode; - protected JsonNode exampleAzureJsonNode; - protected JsonNode exampleGCSJsonNode; - protected List exampleNodes; + private static final ObjectMapper mapper = new ObjectMapper(); + + static JsonNode exampleS3JsonNode; + static JsonNode exampleS3StageEndpointJsonNode; + static JsonNode exampleAzureJsonNode; + static JsonNode exampleGCSJsonNode; + static JsonNode exampleGCSJsonNodeWithUseRegionalUrl; + static JsonNode exampleGCSJsonNodeWithEndPoint; + static List exampleNodes; + + private static JsonNode readJsonFromFile(String name) throws IOException { + try (InputStream is = + FileUploaderPrep.class.getResourceAsStream("/FileUploaderPrep/" + name + ".json")) { + return mapper.readTree(is); + } + } - @Before - public void setup() throws Exception { - exampleS3JsonNode = mapper.readTree(exampleS3JsonString); - exampleS3StageEndpointJsonNode = mapper.readTree(exampleS3JsonStringWithStageEndpoint); - exampleAzureJsonNode = mapper.readTree(exampleAzureJsonString); - exampleGCSJsonNode = mapper.readTree(exampleGCSJsonString); - exampleNodes = Arrays.asList(exampleS3JsonNode, exampleAzureJsonNode, exampleGCSJsonNode); + @BeforeClass + public static void setup() throws Exception { + exampleS3JsonNode = readJsonFromFile("exampleS3"); + exampleS3StageEndpointJsonNode = readJsonFromFile("exampleS3WithStageEndpoint"); + exampleAzureJsonNode = readJsonFromFile("exampleAzure"); + exampleGCSJsonNode = readJsonFromFile("exampleGCS"); + exampleGCSJsonNodeWithUseRegionalUrl = readJsonFromFile("exampleGCSWithUseRegionalUrl"); + exampleGCSJsonNodeWithEndPoint = readJsonFromFile("exampleGCSWithEndpoint"); + exampleNodes = + Arrays.asList( + exampleS3JsonNode, + exampleAzureJsonNode, + exampleGCSJsonNode, + exampleGCSJsonNodeWithUseRegionalUrl, + exampleGCSJsonNodeWithEndPoint); } } diff --git a/src/test/java/net/snowflake/client/jdbc/FileUploaderSessionlessTest.java b/src/test/java/net/snowflake/client/jdbc/FileUploaderSessionlessTest.java index e23800e4e..f5fb7f719 100644 --- a/src/test/java/net/snowflake/client/jdbc/FileUploaderSessionlessTest.java +++ b/src/test/java/net/snowflake/client/jdbc/FileUploaderSessionlessTest.java @@ -3,6 +3,11 @@ */ package net.snowflake.client.jdbc; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -11,6 +16,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.common.core.RemoteStoreFileEncryptionMaterial; import org.junit.Assert; @@ -265,6 +271,7 @@ public void testGetFileTransferMetadatasGCS() throws Exception { Assert.assertEquals(null, stageInfo.getEndPoint()); Assert.assertEquals(null, stageInfo.getStorageAccount()); Assert.assertEquals(true, stageInfo.getIsClientSideEncrypted()); + assertEquals(Optional.empty(), stageInfo.gcsCustomEndpoint()); // EncryptionMaterial check Assert.assertEquals("EXAMPLE_QUERY_ID", metadata.getEncryptionMaterial().getQueryId()); @@ -279,12 +286,42 @@ public void testGetFileTransferMetadatasGCS() throws Exception { Assert.assertEquals("orders_100.csv", metadata.getPresignedUrlFileName()); } + @Test + public void testGetFileTransferMetadataGCSWithUseRegionalUrl() throws Exception { + List metadataList = + SnowflakeFileTransferAgent.getFileTransferMetadatas(exampleGCSJsonNodeWithUseRegionalUrl); + Assert.assertEquals(1, metadataList.size()); + + SnowflakeFileTransferMetadataV1 metadata = + (SnowflakeFileTransferMetadataV1) metadataList.get(0); + + StageInfo stageInfo = metadata.getStageInfo(); + + assertTrue(stageInfo.getUseRegionalUrl()); + assertEquals(Optional.of("storage.us-west1.rep.googleapis.com"), stageInfo.gcsCustomEndpoint()); + } + + @Test + public void testGetFileTransferMetadataGCSWithEndPoint() throws Exception { + List metadataList = + SnowflakeFileTransferAgent.getFileTransferMetadatas(exampleGCSJsonNodeWithEndPoint); + Assert.assertEquals(1, metadataList.size()); + + SnowflakeFileTransferMetadataV1 metadata = + (SnowflakeFileTransferMetadataV1) metadataList.get(0); + + StageInfo stageInfo = metadata.getStageInfo(); + + assertFalse(stageInfo.getUseRegionalUrl()); + assertEquals(Optional.of("example.com"), stageInfo.gcsCustomEndpoint()); + } + @Test public void testGetFileTransferMetadatasUploadError() throws Exception { JsonNode downloadNode = mapper.readTree("{\"data\": {\"command\": \"DOWNLOAD\"}}"); try { SnowflakeFileTransferAgent.getFileTransferMetadatas(downloadNode); - Assert.assertTrue(false); + fail(); } catch (SnowflakeSQLException err) { Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode()); Assert.assertEquals( @@ -297,10 +334,10 @@ public void testGetFileTransferMetadatasEncryptionMaterialError() throws Excepti JsonNode garbageNode = mapper.readTree("{\"data\": {\"src_locations\": [1, 2]}}"); try { SnowflakeFileTransferAgent.getFileTransferMetadatas(garbageNode); - Assert.assertTrue(false); + fail(); } catch (SnowflakeSQLException err) { Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode()); - Assert.assertTrue( + assertTrue( err.getMessage().contains("JDBC driver internal error: Failed to parse the credentials")); } } @@ -312,11 +349,10 @@ public void testGetFileTransferMetadatasUnsupportedLocationError() throws Except foo.put("locationType", "LOCAL_FS"); try { SnowflakeFileTransferAgent.getFileTransferMetadatas(modifiedNode); - Assert.assertTrue(false); + fail(); } catch (SnowflakeSQLException err) { Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode()); - Assert.assertTrue( - err.getMessage().contains("JDBC driver internal error: This API only supports")); + assertTrue(err.getMessage().contains("JDBC driver internal error: This API only supports")); } } @@ -325,10 +361,10 @@ public void testGetFileTransferMetadatasSrcLocationsArrayError() throws JsonProc JsonNode garbageNode = mapper.readTree("{\"data\": {\"src_locations\": \"abc\"}}"); try { SnowflakeFileTransferAgent.getFileTransferMetadatas(garbageNode); - Assert.assertTrue(false); + fail(); } catch (SnowflakeSQLException err) { Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode()); - Assert.assertTrue( + assertTrue( err.getMessage().contains("JDBC driver internal error: src_locations must be an array")); } } @@ -340,10 +376,10 @@ public void testGetFileMetadatasEncryptionMaterialsException() { foo.put("encryptionMaterial", "[1, 2, 3]]"); try { SnowflakeFileTransferAgent.getFileTransferMetadatas(modifiedNode); - Assert.assertTrue(false); + fail(); } catch (SnowflakeSQLException err) { Assert.assertEquals((long) ErrorCode.INTERNAL_ERROR.getMessageCode(), err.getErrorCode()); - Assert.assertTrue(err.getMessage().contains("Failed to parse encryptionMaterial")); + assertTrue(err.getMessage().contains("Failed to parse encryptionMaterial")); } } } diff --git a/src/test/java/net/snowflake/client/jdbc/cloud/storage/StageInfoGcsCustomEndpointTest.java b/src/test/java/net/snowflake/client/jdbc/cloud/storage/StageInfoGcsCustomEndpointTest.java new file mode 100644 index 000000000..f8e00d7eb --- /dev/null +++ b/src/test/java/net/snowflake/client/jdbc/cloud/storage/StageInfoGcsCustomEndpointTest.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved. + */ +package net.snowflake.client.jdbc.cloud.storage; + +import static org.junit.Assert.assertEquals; + +import java.util.HashMap; +import java.util.Optional; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class StageInfoGcsCustomEndpointTest { + private final String region; + private final boolean useRegionalUrl; + private final String endPoint; + private final Optional expectedHost; + + public StageInfoGcsCustomEndpointTest( + String region, boolean useRegionalUrl, String endPoint, Optional expectedHost) { + this.region = region; + this.useRegionalUrl = useRegionalUrl; + this.endPoint = endPoint; + this.expectedHost = expectedHost; + } + + @Test + public void shouldReturnEmptyGCSRegionalUrlWhenNotMeCentral1AndNotUseRegionalUrl() { + StageInfo stageInfo = + StageInfo.createStageInfo("GCS", "bla", new HashMap<>(), region, endPoint, "account", true); + stageInfo.setUseRegionalUrl(useRegionalUrl); + assertEquals(expectedHost, stageInfo.gcsCustomEndpoint()); + } + + @Parameterized.Parameters() + public static Object[][] data() { + return new Object[][] { + {"US-CENTRAL1", false, null, Optional.empty()}, + {"US-CENTRAL1", false, "", Optional.empty()}, + {"US-CENTRAL1", false, "null", Optional.empty()}, + {"US-CENTRAL1", false, " ", Optional.empty()}, + {"US-CENTRAL1", false, "example.com", Optional.of("example.com")}, + {"ME-CENTRAL2", false, null, Optional.of("storage.me-central2.rep.googleapis.com")}, + {"ME-CENTRAL2", true, null, Optional.of("storage.me-central2.rep.googleapis.com")}, + {"ME-CENTRAL2", true, "", Optional.of("storage.me-central2.rep.googleapis.com")}, + {"ME-CENTRAL2", true, " ", Optional.of("storage.me-central2.rep.googleapis.com")}, + {"ME-CENTRAL2", true, "example.com", Optional.of("example.com")}, + {"US-CENTRAL1", true, null, Optional.of("storage.us-central1.rep.googleapis.com")}, + {"US-CENTRAL1", true, "", Optional.of("storage.us-central1.rep.googleapis.com")}, + {"US-CENTRAL1", true, " ", Optional.of("storage.us-central1.rep.googleapis.com")}, + {"US-CENTRAL1", true, "null", Optional.of("storage.us-central1.rep.googleapis.com")}, + {"US-CENTRAL1", true, "example.com", Optional.of("example.com")}, + }; + } +} diff --git a/src/test/resources/FileUploaderPrep/exampleAzure.json b/src/test/resources/FileUploaderPrep/exampleAzure.json new file mode 100644 index 000000000..a2b1835c3 --- /dev/null +++ b/src/test/resources/FileUploaderPrep/exampleAzure.json @@ -0,0 +1,51 @@ +{ + "data": { + "uploadInfo": { + "locationType": "AZURE", + "location": "EXAMPLE_LOCATION/", + "path": "EXAMPLE_PATH/", + "region": "westus", + "storageAccount": "sfcdev2stage", + "isClientSideEncrypted": true, + "creds": { + "AZURE_SAS_TOKEN": "EXAMPLE_AZURE_SAS_TOKEN" + }, + "presignedUrl": null, + "endPoint": "blob.core.windows.net" + }, + "src_locations": [ + "/foo/orders_100.csv" + ], + "parallel": 4, + "threshold": 209715200, + "autoCompress": true, + "overwrite": false, + "sourceCompression": "auto_detect", + "clientShowEncryptionParameter": false, + "queryId": "EXAMPLE_QUERY_ID", + "encryptionMaterial": { + "queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY", + "queryId": "EXAMPLE_QUERY_ID", + "smkId": 123 + }, + "stageInfo": { + "locationType": "AZURE", + "location": "EXAMPLE_LOCATION/", + "path": "EXAMPLE_PATH/", + "region": "westus", + "storageAccount": "EXAMPLE_STORAGE_ACCOUNT", + "isClientSideEncrypted": true, + "creds": { + "AZURE_SAS_TOKEN": "EXAMPLE_AZURE_SAS_TOKEN" + }, + "presignedUrl": null, + "endPoint": "blob.core.windows.net" + }, + "command": "UPLOAD", + "kind": null, + "operation": "Node" + }, + "code": null, + "message": null, + "success": true +} \ No newline at end of file diff --git a/src/test/resources/FileUploaderPrep/exampleGCS.json b/src/test/resources/FileUploaderPrep/exampleGCS.json new file mode 100644 index 000000000..8cd605f1c --- /dev/null +++ b/src/test/resources/FileUploaderPrep/exampleGCS.json @@ -0,0 +1,47 @@ +{ + "data": { + "uploadInfo": { + "locationType": "GCS", + "location": "foo/tables/9224/", + "path": "tables/9224/", + "region": "US-WEST1", + "storageAccount": "", + "isClientSideEncrypted": true, + "creds": {}, + "presignedUrl": "EXAMPLE_PRESIGNED_URL", + "endPoint": "" + }, + "src_locations": [ + "/foo/bart/orders_100.csv" + ], + "parallel": 4, + "threshold": 209715200, + "autoCompress": true, + "overwrite": false, + "sourceCompression": "auto_detect", + "clientShowEncryptionParameter": false, + "queryId": "EXAMPLE_QUERY_ID", + "encryptionMaterial": { + "queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY", + "queryId": "EXAMPLE_QUERY_ID", + "smkId": 123 + }, + "stageInfo": { + "locationType": "GCS", + "location": "foo/tables/9224/", + "path": "tables/9224/", + "region": "US-WEST1", + "storageAccount": "", + "isClientSideEncrypted": true, + "creds": {}, + "presignedUrl": "EXAMPLE_PRESIGNED_URL", + "endPoint": "" + }, + "command": "UPLOAD", + "kind": null, + "operation": "Node" + }, + "code": null, + "message": null, + "success": true +} \ No newline at end of file diff --git a/src/test/resources/FileUploaderPrep/exampleGCSWithEndpoint.json b/src/test/resources/FileUploaderPrep/exampleGCSWithEndpoint.json new file mode 100644 index 000000000..8ba946c76 --- /dev/null +++ b/src/test/resources/FileUploaderPrep/exampleGCSWithEndpoint.json @@ -0,0 +1,47 @@ +{ + "data": { + "uploadInfo": { + "locationType": "GCS", + "location": "foo/tables/9224/", + "path": "tables/9224/", + "region": "US-WEST1", + "storageAccount": "", + "isClientSideEncrypted": true, + "creds": {}, + "presignedUrl": "EXAMPLE_PRESIGNED_URL", + "endPoint": "example.com" + }, + "src_locations": [ + "/foo/bart/orders_100.csv" + ], + "parallel": 4, + "threshold": 209715200, + "autoCompress": true, + "overwrite": false, + "sourceCompression": "auto_detect", + "clientShowEncryptionParameter": false, + "queryId": "EXAMPLE_QUERY_ID", + "encryptionMaterial": { + "queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY", + "queryId": "EXAMPLE_QUERY_ID", + "smkId": 123 + }, + "stageInfo": { + "locationType": "GCS", + "location": "foo/tables/9224/", + "path": "tables/9224/", + "region": "US-WEST1", + "storageAccount": "", + "isClientSideEncrypted": true, + "creds": {}, + "presignedUrl": "EXAMPLE_PRESIGNED_URL", + "endPoint": "example.com" + }, + "command": "UPLOAD", + "kind": null, + "operation": "Node" + }, + "code": null, + "message": null, + "success": true +} \ No newline at end of file diff --git a/src/test/resources/FileUploaderPrep/exampleGCSWithUseRegionalUrl.json b/src/test/resources/FileUploaderPrep/exampleGCSWithUseRegionalUrl.json new file mode 100644 index 000000000..79f4dc678 --- /dev/null +++ b/src/test/resources/FileUploaderPrep/exampleGCSWithUseRegionalUrl.json @@ -0,0 +1,49 @@ +{ + "data": { + "uploadInfo": { + "locationType": "GCS", + "useRegionalUrl": true, + "location": "foo/tables/9224/", + "path": "tables/9224/", + "region": "US-WEST1", + "storageAccount": "", + "isClientSideEncrypted": true, + "creds": {}, + "presignedUrl": "EXAMPLE_PRESIGNED_URL", + "endPoint": "" + }, + "src_locations": [ + "/foo/bart/orders_100.csv" + ], + "parallel": 4, + "threshold": 209715200, + "autoCompress": true, + "overwrite": false, + "sourceCompression": "auto_detect", + "clientShowEncryptionParameter": false, + "queryId": "EXAMPLE_QUERY_ID", + "encryptionMaterial": { + "queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY", + "queryId": "EXAMPLE_QUERY_ID", + "smkId": 123 + }, + "stageInfo": { + "locationType": "GCS", + "useRegionalUrl": true, + "location": "foo/tables/9224/", + "path": "tables/9224/", + "region": "US-WEST1", + "storageAccount": "", + "isClientSideEncrypted": true, + "creds": {}, + "presignedUrl": "EXAMPLE_PRESIGNED_URL", + "endPoint": "" + }, + "command": "UPLOAD", + "kind": null, + "operation": "Node" + }, + "code": null, + "message": null, + "success": true +} \ No newline at end of file diff --git a/src/test/resources/FileUploaderPrep/exampleS3.json b/src/test/resources/FileUploaderPrep/exampleS3.json new file mode 100644 index 000000000..eadc166d8 --- /dev/null +++ b/src/test/resources/FileUploaderPrep/exampleS3.json @@ -0,0 +1,60 @@ +{ + "data": { + "uploadInfo": { + "locationType": "S3", + "location": "example/location", + "path": "tables/19805757505/", + "region": "us-west-2", + "storageAccount": null, + "isClientSideEncrypted": true, + "creds": { + "AWS_KEY_ID": "EXAMPLE_AWS_KEY_ID", + "AWS_SECRET_KEY": "EXAMPLE_AWS_SECRET_KEY", + "AWS_TOKEN": "EXAMPLE_AWS_TOKEN", + "AWS_ID": "EXAMPLE_AWS_ID", + "AWS_KEY": "EXAMPLE_AWS_KEY" + }, + "presignedUrl": null, + "endPoint": null + }, + "src_locations": [ + "/tmp/files/orders_100.csv" + ], + "parallel": 4, + "threshold": 209715200, + "autoCompress": true, + "overwrite": false, + "sourceCompression": "auto_detect", + "clientShowEncryptionParameter": true, + "queryId": "EXAMPLE_QUERY_ID", + "encryptionMaterial": { + "queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY", + "queryId": "EXAMPLE_QUERY_ID", + "smkId": 123 + }, + "stageInfo": { + "locationType": "S3", + "location": "stage/location/foo/", + "path": "tables/19805757505/", + "region": "us-west-2", + "storageAccount": null, + "isClientSideEncrypted": true, + "useS3RegionalUrl": true, + "creds": { + "AWS_KEY_ID": "EXAMPLE_AWS_KEY_ID", + "AWS_SECRET_KEY": "EXAMPLE_AWS_SECRET_KEY", + "AWS_TOKEN": "EXAMPLE_AWS_TOKEN", + "AWS_ID": "EXAMPLE_AWS_ID", + "AWS_KEY": "EXAMPLE_AWS_KEY" + }, + "presignedUrl": null, + "endPoint": null + }, + "command": "UPLOAD", + "kind": null, + "operation": "Node" + }, + "code": null, + "message": null, + "success": true +} \ No newline at end of file diff --git a/src/test/resources/FileUploaderPrep/exampleS3WithStageEndpoint.json b/src/test/resources/FileUploaderPrep/exampleS3WithStageEndpoint.json new file mode 100644 index 000000000..32b8a66a1 --- /dev/null +++ b/src/test/resources/FileUploaderPrep/exampleS3WithStageEndpoint.json @@ -0,0 +1,59 @@ +{ + "data": { + "uploadInfo": { + "locationType": "S3", + "location": "example/location", + "path": "tables/19805757505/", + "region": "us-west-2", + "storageAccount": null, + "isClientSideEncrypted": true, + "creds": { + "AWS_KEY_ID": "EXAMPLE_AWS_KEY_ID", + "AWS_SECRET_KEY": "EXAMPLE_AWS_SECRET_KEY", + "AWS_TOKEN": "EXAMPLE_AWS_TOKEN", + "AWS_ID": "EXAMPLE_AWS_ID", + "AWS_KEY": "EXAMPLE_AWS_KEY" + }, + "presignedUrl": null, + "endPoint": null + }, + "src_locations": [ + "/tmp/files/orders_100.csv" + ], + "parallel": 4, + "threshold": 209715200, + "autoCompress": true, + "overwrite": false, + "sourceCompression": "auto_detect", + "clientShowEncryptionParameter": true, + "queryId": "EXAMPLE_QUERY_ID", + "encryptionMaterial": { + "queryStageMasterKey": "EXAMPLE_QUERY_STAGE_MASTER_KEY", + "queryId": "EXAMPLE_QUERY_ID", + "smkId": 123 + }, + "stageInfo": { + "locationType": "S3", + "location": "stage/location/foo/", + "path": "tables/19805757505/", + "region": "us-west-2", + "storageAccount": null, + "isClientSideEncrypted": true, + "creds": { + "AWS_KEY_ID": "EXAMPLE_AWS_KEY_ID", + "AWS_SECRET_KEY": "EXAMPLE_AWS_SECRET_KEY", + "AWS_TOKEN": "EXAMPLE_AWS_TOKEN", + "AWS_ID": "EXAMPLE_AWS_ID", + "AWS_KEY": "EXAMPLE_AWS_KEY" + }, + "presignedUrl": null, + "endPoint": "s3-fips.us-east-1.amazonaws.com" + }, + "command": "UPLOAD", + "kind": null, + "operation": "Node" + }, + "code": null, + "message": null, + "success": true +} \ No newline at end of file