diff --git a/.github/workflows/End2EndTest.yml b/.github/workflows/End2EndTest.yml index a98c97fd2..91b16125e 100644 --- a/.github/workflows/End2EndTest.yml +++ b/.github/workflows/End2EndTest.yml @@ -67,3 +67,26 @@ jobs: - name: Unit & Integration Test (Windows) continue-on-error: false run: mvn -DghActionsIT verify --batch-mode + build-e2e-jar-test: + name: E2E JAR Test - ${{ matrix.java }}, Cloud ${{ matrix.snowflake_cloud }} + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + java: [ 8 ] + snowflake_cloud: [ 'AWS' ] + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Install Java ${{ matrix.java }} + uses: actions/setup-java@v2 + with: + distribution: temurin + java-version: ${{ matrix.java }} + cache: maven + - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }} + env: + DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }} + run: ./scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }} + - name: Run E2E JAR Test + run: ./e2e-jar-test/run_e2e_jar_test.sh diff --git a/README.md b/README.md index edbded16d..a29e9febe 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,25 @@ dependencies { } ``` +## Jar Versions + +The Snowflake Ingest SDK provides shaded and unshaded versions of its jar. The shaded version bundles the dependencies into its own jar, +whereas the unshaded version declares its dependencies in `pom.xml`, which are fetched as standard transitive dependencies by the build system like Maven or Gradle. +The shaded JAR can help avoid potential dependency conflicts, but the unshaded version provides finer graned control over transitive dependencies. + +## Using with snowflake-jdbc-fics + +For use cases, which need to use `snowflake-jdbc-fips` instead of the default `snowflake-jdbc`, we recommend to take the following steps: + +- Use the unshaded version of the Ingest SDK. +- Exclude these transitive dependencies: + - `net.snowflake:snowflake-jdbc` + - `org.bouncycastle:bcpkix-jdk18on` + - `org.bouncycastle:bcprov-jdk18on` +- Add a dependency on `snowflake-jdbc-fips`. + +See [this test](https://github.com/snowflakedb/snowflake-ingest-java/tree/master/e2e-jar-test/fips) for an example how to use Snowflake Ingest SDK together with Snowflake FIPS JDBC Driver. + # Example ## Snowpipe diff --git a/e2e-jar-test/core/pom.xml b/e2e-jar-test/core/pom.xml new file mode 100644 index 000000000..36d52ae5b --- /dev/null +++ b/e2e-jar-test/core/pom.xml @@ -0,0 +1,38 @@ + + 4.0.0 + + net.snowflake.snowflake-ingest-java-e2e-jar-test + parent + 1.0-SNAPSHOT + + + core + core + jar + + + UTF-8 + 1.8 + 1.8 + + + + + + net.snowflake + snowflake-ingest-sdk + provided + + + + org.slf4j + slf4j-simple + + + + com.fasterxml.jackson.core + jackson-databind + + + diff --git a/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java b/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java new file mode 100644 index 000000000..100972ea0 --- /dev/null +++ b/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java @@ -0,0 +1,169 @@ +package net.snowflake; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.PKCS8EncodedKeySpec; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Base64; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.UUID; + +import net.snowflake.ingest.streaming.OpenChannelRequest; +import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel; +import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient; +import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory; + +public class IngestTestUtils { + private static final String PROFILE_PATH = "profile.json"; + + private final Connection connection; + + private final String database; + private final String schema; + private final String table; + + private final String testId; + + private final SnowflakeStreamingIngestClient client; + + private final SnowflakeStreamingIngestChannel channel; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + private final Random random = new Random(); + + private final Base64.Decoder base64Decoder = Base64.getDecoder(); + + public IngestTestUtils(String testName) + throws SQLException, + IOException, + ClassNotFoundException, + NoSuchAlgorithmException, + InvalidKeySpecException { + testId = String.format("%s_%s", testName, UUID.randomUUID().toString().replace("-", "_")); + connection = getConnection(); + database = String.format("database_%s", testId); + schema = String.format("schema_%s", testId); + table = String.format("table_%s", testId); + + connection.createStatement().execute(String.format("create database %s", database)); + connection.createStatement().execute(String.format("create schema %s", schema)); + connection.createStatement().execute(String.format("create table %s (c1 int, c2 varchar, c3 binary)", table)); + + client = + SnowflakeStreamingIngestClientFactory.builder("TestClient01") + .setProperties(loadProperties()) + .build(); + + channel = client.openChannel( + OpenChannelRequest.builder(String.format("channel_%s", this.testId)) + .setDBName(database) + .setSchemaName(schema) + .setTableName(table) + .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE) + .build()); + } + + private Properties loadProperties() throws IOException { + Properties props = new Properties(); + Iterator> propIt = + objectMapper.readTree(new String(Files.readAllBytes(Paths.get(PROFILE_PATH)))).fields(); + while (propIt.hasNext()) { + Map.Entry prop = propIt.next(); + props.put(prop.getKey(), prop.getValue().asText()); + } + return props; + } + + private Connection getConnection() + throws IOException, ClassNotFoundException, SQLException, NoSuchAlgorithmException, InvalidKeySpecException { + Class.forName("net.snowflake.client.jdbc.SnowflakeDriver"); + + Properties loadedProps = loadProperties(); + + byte[] decoded = base64Decoder.decode(loadedProps.getProperty("private_key")); + KeyFactory kf = KeyFactory.getInstance("RSA"); + + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(decoded); + PrivateKey privateKey = kf.generatePrivate(keySpec); + + Properties props = new Properties(); + props.putAll(loadedProps); + props.put("client_session_keep_alive", "true"); + props.put("privateKey", privateKey); + + return DriverManager.getConnection(loadedProps.getProperty("connect_string"), props); + } + + private Map createRow() { + Map row = new HashMap<>(); + + byte[] bytes = new byte[1024]; + random.nextBytes(bytes); + + row.put("c1", random.nextInt()); + row.put("c2", String.valueOf(random.nextInt())); + row.put("c3", bytes); + + return row; + } + + /** + * Given a channel and expected offset, this method waits up to 60 seconds until the last + * committed offset is equal to the passed offset + */ + private void waitForOffset(SnowflakeStreamingIngestChannel channel, String expectedOffset) + throws InterruptedException { + int counter = 0; + String lastCommittedOffset = null; + while (counter < 600) { + String currentOffset = channel.getLatestCommittedOffsetToken(); + if (expectedOffset.equals(currentOffset)) { + return; + } + System.out.printf("Waiting for offset expected=%s actual=%s%n", expectedOffset, currentOffset); + lastCommittedOffset = currentOffset; + counter++; + Thread.sleep(100); + } + throw new RuntimeException( + String.format( + "Timeout exceeded while waiting for offset %s. Last committed offset: %s", + expectedOffset, lastCommittedOffset)); + } + + public void test() throws InterruptedException { + // Insert few rows one by one + for (int offset = 2; offset < 1000; offset++) { + offset++; + channel.insertRow(createRow(), String.valueOf(offset)); + } + + // Insert a batch of rows + String offset = "final-offset"; + channel.insertRows( + Arrays.asList(createRow(), createRow(), createRow(), createRow(), createRow()), offset); + + waitForOffset(channel, offset); + } + + public void close() throws Exception { + connection.close(); + channel.close().get(); + client.close(); + } +} diff --git a/e2e-jar-test/fips/pom.xml b/e2e-jar-test/fips/pom.xml new file mode 100644 index 000000000..039af9b7a --- /dev/null +++ b/e2e-jar-test/fips/pom.xml @@ -0,0 +1,53 @@ + + 4.0.0 + + net.snowflake.snowflake-ingest-java-e2e-jar-test + parent + 1.0-SNAPSHOT + + + fips + jar + fips + + + UTF-8 + 1.8 + 1.8 + + + + + net.snowflake.snowflake-ingest-java-e2e-jar-test + core + + + + net.snowflake + snowflake-ingest-sdk + + + net.snowflake + snowflake-jdbc + + + org.bouncycastle + bcpkix-jdk18on + + + org.bouncycastle + bcprov-jdk18on + + + + + net.snowflake + snowflake-jdbc-fips + + + junit + junit + + + diff --git a/e2e-jar-test/fips/src/test/java/net/snowflake/FipsIngestE2ETest.java b/e2e-jar-test/fips/src/test/java/net/snowflake/FipsIngestE2ETest.java new file mode 100644 index 000000000..7279f23ff --- /dev/null +++ b/e2e-jar-test/fips/src/test/java/net/snowflake/FipsIngestE2ETest.java @@ -0,0 +1,31 @@ +package net.snowflake; + +import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.security.Security; + +public class FipsIngestE2ETest { + + private IngestTestUtils ingestTestUtils; + + @Before + public void setUp() throws Exception { + // Add FIPS provider, the SDK does not do this by default + Security.addProvider(new BouncyCastleFipsProvider("C:HYBRID;ENABLE{All};")); + + ingestTestUtils = new IngestTestUtils("fips_ingest"); + } + + @After + public void tearDown() throws Exception { + ingestTestUtils.close(); + } + + @Test + public void name() throws InterruptedException { + ingestTestUtils.test(); + } +} diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml new file mode 100644 index 000000000..9368308fc --- /dev/null +++ b/e2e-jar-test/pom.xml @@ -0,0 +1,59 @@ + + 4.0.0 + + net.snowflake.snowflake-ingest-java-e2e-jar-test + parent + 1.0-SNAPSHOT + pom + + snowflake-ingest-sdk-e2e-test + + + standard + fips + core + + + + + + + net.snowflake.snowflake-ingest-java-e2e-jar-test + core + ${project.version} + + + + net.snowflake + snowflake-ingest-sdk + 2.0.4-SNAPSHOT + + + + net.snowflake + snowflake-jdbc-fips + 3.13.30 + + + + org.slf4j + slf4j-simple + 1.7.36 + + + + com.fasterxml.jackson.core + jackson-databind + 2.14.0 + + + + junit + junit + 4.13.2 + test + + + + diff --git a/e2e-jar-test/run_e2e_jar_test.sh b/e2e-jar-test/run_e2e_jar_test.sh new file mode 100755 index 000000000..e5e9b5c55 --- /dev/null +++ b/e2e-jar-test/run_e2e_jar_test.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -euo pipefail + +## This script tests the SDK JARs end-to-end, i.e. not using integration tests from within the project, but from an +## external Maven project, which depends on the SDK deployed into the local maven repository. The following SDK variants are tested: +## 1. Shaded jar +## 2. Unshaded jar +## 3. FIPS-compliant jar, i.e. unshaded jar without snowflake-jdbc and bouncy castle dependencies, but with snowflake-jdbc-fips depedency + +maven_repo_dir=$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) +sdk_repo_dir="${maven_repo_dir}/net/snowflake/snowflake-ingest-sdk" + +cp profile.json e2e-jar-test/standard +cp profile.json e2e-jar-test/fips + +################### +# TEST SHADED JAR # +################### + +# Remove the SDK from local maven repository +rm -fr $sdk_repo_dir + +# Prepare pom.xml for shaded JAR +project_version=$(./scripts/get_project_info_from_pom.py pom.xml version) +./scripts/update_project_version.py public_pom.xml $project_version > generated_public_pom.xml + +# Build shaded SDK +mvn clean package -DskipTests=true --batch-mode --show-version + +# Install shaded SDK JARs into local maven repository +mvn install:install-file -Dfile=target/snowflake-ingest-sdk.jar -DpomFile=generated_public_pom.xml + +# Run e2e tests +(cd e2e-jar-test && mvn clean verify -pl standard -am) + +##################### +# TEST UNSHADED JAR # +##################### + +# Remove the SDK from local maven repository +rm -r $sdk_repo_dir + +# Install unshaded SDK into local maven repository +mvn clean install -Dnot-shadeDep -DskipTests=true --batch-mode --show-version + +# Run e2e tests +(cd e2e-jar-test && mvn clean verify -pl standard -am) + +############# +# TEST FIPS # +############# + +# Remove the SDK from local maven repository +rm -r $sdk_repo_dir + +# Install unshaded SDK into local maven repository +mvn clean install -Dnot-shadeDep -DskipTests=true --batch-mode --show-version + +# Run e2e tests on the FIPS module +(cd e2e-jar-test && mvn clean verify -pl fips -am) \ No newline at end of file diff --git a/e2e-jar-test/standard/pom.xml b/e2e-jar-test/standard/pom.xml new file mode 100644 index 000000000..2fdddcddd --- /dev/null +++ b/e2e-jar-test/standard/pom.xml @@ -0,0 +1,34 @@ + + 4.0.0 + + net.snowflake.snowflake-ingest-java-e2e-jar-test + parent + 1.0-SNAPSHOT + + + standard + jar + unshaded + + + UTF-8 + 1.8 + 1.8 + + + + + net.snowflake.snowflake-ingest-java-e2e-jar-test + core + + + net.snowflake + snowflake-ingest-sdk + + + junit + junit + + + diff --git a/e2e-jar-test/standard/src/test/java/net/snowflake/StandardIngestE2ETest.java b/e2e-jar-test/standard/src/test/java/net/snowflake/StandardIngestE2ETest.java new file mode 100644 index 000000000..255577655 --- /dev/null +++ b/e2e-jar-test/standard/src/test/java/net/snowflake/StandardIngestE2ETest.java @@ -0,0 +1,25 @@ +package net.snowflake; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class StandardIngestE2ETest { + + private IngestTestUtils ingestTestUtils; + + @Before + public void setUp() throws Exception { + ingestTestUtils = new IngestTestUtils("standard_ingest"); + } + + @After + public void tearDown() throws Exception { + ingestTestUtils.close(); + } + + @Test + public void name() throws InterruptedException { + ingestTestUtils.test(); + } +} diff --git a/linkage-checker-exclusion-rules.xml b/linkage-checker-exclusion-rules.xml index 3544c6d48..0cb2eb38c 100644 --- a/linkage-checker-exclusion-rules.xml +++ b/linkage-checker-exclusion-rules.xml @@ -19,6 +19,18 @@ Google Crypto Tink is an optional dependency of nimbus-jose-jwt + + + + + Seems like a false positive, this class does exist on classpath. + + + + + + Seems like a false positive, this class does exist on classpath. + diff --git a/pom.xml b/pom.xml index dbf156690..fc4a2ab81 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,7 @@ + 1.74 1.9.13 1.15 3.2.2 @@ -274,6 +275,16 @@ audience-annotations ${yetus.version} + + org.bouncycastle + bcpkix-jdk18on + ${bouncycastle.version} + + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.version} + org.codehaus.jackson jackson-core-asl @@ -449,6 +460,15 @@ org.apache.parquet parquet-hadoop + + + org.bouncycastle + bcpkix-jdk18on + + + org.bouncycastle + bcprov-jdk18on + org.slf4j slf4j-api @@ -531,7 +551,7 @@ com.google.cloud.tools linkage-checker-enforcer-rules - 1.5.12 + 1.5.13 org.codehaus.mojo @@ -758,6 +778,7 @@ The MIT License EDL 1.0 The Go license + Bouncy Castle Licence test,provided,system true @@ -929,6 +950,10 @@ com.nimbusds ${shadeBase}.com.nimbusds + + org.bouncycastle + ${shadeBase}.org.bouncycastle + net.jcip ${shadeBase}.net.jcip diff --git a/scripts/process_licenses.py b/scripts/process_licenses.py index a17c34843..bb43fbbf0 100644 --- a/scripts/process_licenses.py +++ b/scripts/process_licenses.py @@ -30,6 +30,7 @@ EDL_10_LICENSE = "EDL 1.0" MIT_LICENSE = "The MIT License" GO_LICENSE = "The Go license" +BOUNCY_CASTLE_LICENSE = "Bouncy Castle Licence " # The SDK does not need to include licenses of dependencies, which aren't shaded IGNORED_DEPENDENCIES = {"net.snowflake:snowflake-jdbc", "org.slf4j:slf4j-api"} @@ -57,6 +58,9 @@ "org.apache.parquet:parquet-format-structures": APACHE_LICENSE, "com.github.luben:zstd-jni": BSD_2_CLAUSE_LICENSE, "io.airlift:aircompressor": APACHE_LICENSE, + "org.bouncycastle:bcpkix-jdk18on": BOUNCY_CASTLE_LICENSE, + "org.bouncycastle:bcutil-jdk18on": BOUNCY_CASTLE_LICENSE, + "org.bouncycastle:bcprov-jdk18on": BOUNCY_CASTLE_LICENSE, } diff --git a/scripts/update_project_version.py b/scripts/update_project_version.py index dd844a7c5..c67d58802 100755 --- a/scripts/update_project_version.py +++ b/scripts/update_project_version.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # -*- coding: utf-8 -*- # # Update project version diff --git a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java index 97bdae0f1..b212d7244 100644 --- a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java +++ b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java @@ -40,7 +40,8 @@ public enum ErrorCode { MAKE_URI_FAILURE("0032"), OAUTH_REFRESH_TOKEN_ERROR("0033"), INVALID_CONFIG_PARAMETER("0034"), - MAX_BATCH_SIZE_EXCEEDED("0035"); + MAX_BATCH_SIZE_EXCEEDED("0035"), + CRYPTO_PROVIDER_ERROR("0036"); public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages"; diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java index ca39fed0d..a2653ab14 100644 --- a/src/main/java/net/snowflake/ingest/utils/Utils.java +++ b/src/main/java/net/snowflake/ingest/utils/Utils.java @@ -12,10 +12,12 @@ import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; import java.lang.management.MemoryUsage; +import java.lang.reflect.InvocationTargetException; import java.security.KeyFactory; import java.security.KeyPair; import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; +import java.security.Provider; import java.security.PublicKey; import java.security.Security; import java.security.interfaces.RSAPrivateCrtKey; @@ -26,20 +28,66 @@ import java.util.Map; import java.util.Properties; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.pkcs.PrivateKeyInfo; -import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider; -import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.PEMParser; -import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; -import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder; -import net.snowflake.client.jdbc.internal.org.bouncycastle.operator.InputDecryptorProvider; -import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo; import org.apache.commons.codec.binary.Base64; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; +import org.bouncycastle.openssl.PEMParser; +import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; +import org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder; +import org.bouncycastle.operator.InputDecryptorProvider; +import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo; /** Contains Ingest related utility functions */ public class Utils { private static final Logging logger = new Logging(Utils.class); + private static final String DEFAULT_SECURITY_PROVIDER_NAME = + "org.bouncycastle.jce.provider.BouncyCastleProvider"; + + /** provider name */ + private static final String BOUNCY_CASTLE_PROVIDER = "BC"; + /** provider name for FIPS */ + private static final String BOUNCY_CASTLE_FIPS_PROVIDER = "BCFIPS"; + + static { + // Add Bouncy Castle to the security provider. This is required to + // verify the signature on OCSP response and attached certificates. + if (Security.getProvider(BOUNCY_CASTLE_PROVIDER) == null + && Security.getProvider(BOUNCY_CASTLE_FIPS_PROVIDER) == null) { + Security.addProvider(instantiateSecurityProvider()); + } + } + + public static Provider getProvider() { + final Provider bcProvider = Security.getProvider(BOUNCY_CASTLE_PROVIDER); + if (bcProvider != null) { + return bcProvider; + } + final Provider bcFipsProvider = Security.getProvider(BOUNCY_CASTLE_FIPS_PROVIDER); + if (bcFipsProvider != null) { + return bcFipsProvider; + } + throw new SFException(ErrorCode.INTERNAL_ERROR, "No security provider found"); + } + + private static Provider instantiateSecurityProvider() { + try { + logger.logInfo("Adding security provider {}", DEFAULT_SECURITY_PROVIDER_NAME); + Class klass = Class.forName(DEFAULT_SECURITY_PROVIDER_NAME); + return (Provider) klass.getDeclaredConstructor().newInstance(); + } catch (ExceptionInInitializerError + | ClassNotFoundException + | NoSuchMethodException + | InstantiationException + | IllegalAccessException + | IllegalArgumentException + | InvocationTargetException + | SecurityException ex) { + throw new SFException( + ErrorCode.CRYPTO_PROVIDER_ERROR, DEFAULT_SECURITY_PROVIDER_NAME, ex.getMessage()); + } + } + /** * Assert when the String is null or Empty * @@ -183,7 +231,6 @@ public static PrivateKey parsePrivateKey(String key) { key = key.replaceAll("-+[A-Za-z ]+-+", ""); key = key.replaceAll("\\s", ""); - java.security.Security.addProvider(new BouncyCastleProvider()); byte[] encoded = Base64.decodeBase64(key); try { KeyFactory kf = KeyFactory.getInstance("RSA"); @@ -216,7 +263,6 @@ public static PrivateKey parseEncryptedPrivateKey(String key, String passphrase) } builder.append("\n-----END ENCRYPTED PRIVATE KEY-----"); key = builder.toString(); - Security.addProvider(new BouncyCastleProvider()); try { PEMParser pemParser = new PEMParser(new StringReader(key)); PKCS8EncryptedPrivateKeyInfo encryptedPrivateKeyInfo = @@ -225,7 +271,7 @@ public static PrivateKey parseEncryptedPrivateKey(String key, String passphrase) InputDecryptorProvider pkcs8Prov = new JceOpenSSLPKCS8DecryptorProviderBuilder().build(passphrase.toCharArray()); JcaPEMKeyConverter converter = - new JcaPEMKeyConverter().setProvider(BouncyCastleProvider.PROVIDER_NAME); + new JcaPEMKeyConverter().setProvider(Utils.getProvider().getName()); PrivateKeyInfo decryptedPrivateKeyInfo = encryptedPrivateKeyInfo.decryptPrivateKeyInfo(pkcs8Prov); return converter.getPrivateKey(decryptedPrivateKeyInfo); diff --git a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties index 9698799f7..268afd051 100644 --- a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties +++ b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties @@ -37,4 +37,5 @@ 0032=URI builder fail to build url: {0} 0033=OAuth token refresh failure: {0} 0034=Invalid config parameter: {0} -0035=Too large batch of rows passed to insertRows, the batch size cannot exceed {0} bytes, recommended batch size for optimal performance and memory utilization is {1} bytes. We recommend splitting large batches into multiple smaller ones and call insertRows for each smaller batch separately. \ No newline at end of file +0035=Too large batch of rows passed to insertRows, the batch size cannot exceed {0} bytes, recommended batch size for optimal performance and memory utilization is {1} bytes. We recommend splitting large batches into multiple smaller ones and call insertRows for each smaller batch separately. +0036=Failed to load {0}. If you use FIPS, import BouncyCastleFipsProvider in the application: {1} \ No newline at end of file diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java index b2ebe1bdd..ba14ff610 100644 --- a/src/test/java/net/snowflake/ingest/TestUtils.java +++ b/src/test/java/net/snowflake/ingest/TestUtils.java @@ -41,7 +41,6 @@ import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; -import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider; import net.snowflake.ingest.streaming.InsertValidationResponse; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel; import net.snowflake.ingest.utils.Constants; @@ -123,8 +122,6 @@ private static void init() throws Exception { role = Optional.ofNullable(profile.get(ROLE)).map(r -> r.asText()).orElse("DEFAULT_ROLE"); privateKeyPem = profile.get(PRIVATE_KEY).asText(); - java.security.Security.addProvider(new BouncyCastleProvider()); - byte[] encoded = Base64.decodeBase64(privateKeyPem); KeyFactory kf = KeyFactory.getInstance("RSA"); diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java index 0b8e8b4cd..5a77a51db 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java @@ -39,7 +39,6 @@ import javax.crypto.BadPaddingException; import javax.crypto.IllegalBlockSizeException; import javax.crypto.NoSuchPaddingException; -import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider; import net.snowflake.ingest.streaming.OpenChannelRequest; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.Cryptor; @@ -47,7 +46,6 @@ import net.snowflake.ingest.utils.ParameterProvider; import net.snowflake.ingest.utils.SFException; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; @@ -277,11 +275,6 @@ TestContext>> create() { TestContextFactory testContextFactory; - @Before - public void setup() { - java.security.Security.addProvider(new BouncyCastleProvider()); - } - private SnowflakeStreamingIngestChannelInternal addChannel1(TestContext testContext) { return testContext .channelBuilder("channel1") diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java index ec9f671bb..a99054c9f 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java @@ -19,7 +19,6 @@ import java.io.StringWriter; import java.security.KeyPair; import java.security.PrivateKey; -import java.security.Security; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collections; @@ -40,13 +39,6 @@ import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; import net.snowflake.client.jdbc.internal.google.common.collect.Sets; -import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.nist.NISTObjectIdentifiers; -import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider; -import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JcaPEMWriter; -import net.snowflake.client.jdbc.internal.org.bouncycastle.operator.OperatorCreationException; -import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfoBuilder; -import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.jcajce.JcaPKCS8EncryptedPrivateKeyInfoBuilder; -import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.jcajce.JcePKCSPBEOutputEncryptorBuilder; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.OpenChannelRequest; @@ -59,6 +51,12 @@ import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; +import org.bouncycastle.asn1.nist.NISTObjectIdentifiers; +import org.bouncycastle.openssl.jcajce.JcaPEMWriter; +import org.bouncycastle.operator.OperatorCreationException; +import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfoBuilder; +import org.bouncycastle.pkcs.jcajce.JcaPKCS8EncryptedPrivateKeyInfoBuilder; +import org.bouncycastle.pkcs.jcajce.JcePKCSPBEOutputEncryptorBuilder; import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; @@ -293,7 +291,6 @@ public void testEncryptedPrivateKey() throws Exception { private String generateAESKey(PrivateKey key, char[] passwd) throws IOException, OperatorCreationException { - Security.addProvider(new BouncyCastleProvider()); StringWriter writer = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(writer); PKCS8EncryptedPrivateKeyInfoBuilder pkcs8EncryptedPrivateKeyInfoBuilder = @@ -301,7 +298,7 @@ private String generateAESKey(PrivateKey key, char[] passwd) pemWriter.writeObject( pkcs8EncryptedPrivateKeyInfoBuilder.build( new JcePKCSPBEOutputEncryptorBuilder(NISTObjectIdentifiers.id_aes256_CBC) - .setProvider(BouncyCastleProvider.PROVIDER_NAME) + .setProvider(Utils.getProvider().getName()) .build(passwd))); pemWriter.close(); return writer.toString(); diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java index d9e6e3ddd..2a5023a3a 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java @@ -1,6 +1,6 @@ package net.snowflake.ingest.streaming.internal.datatypes; -import net.snowflake.client.jdbc.internal.org.bouncycastle.util.encoders.Hex; +import org.bouncycastle.util.encoders.Hex; import org.junit.Test; public class BinaryIT extends AbstractDataTypeTest {