From 1648c07051e4fc84a007c19ca9e7a267d58b2158 Mon Sep 17 00:00:00 2001 From: Lukas Sembera Date: Wed, 4 Oct 2023 14:41:44 +0000 Subject: [PATCH] SNOW-926149 Fix issues while using snowflake-jdbc-fips --- linkage-checker-exclusion-rules.xml | 12 ++++ pom.xml | 27 +++++++- scripts/process_licenses.py | 4 ++ .../net/snowflake/ingest/utils/ErrorCode.java | 3 +- .../net/snowflake/ingest/utils/Utils.java | 66 ++++++++++++++++--- .../ingest/ingest_error_messages.properties | 3 +- .../java/net/snowflake/ingest/TestUtils.java | 3 - .../streaming/internal/FlushServiceTest.java | 7 -- .../SnowflakeStreamingIngestClientTest.java | 17 ++--- .../internal/datatypes/BinaryIT.java | 2 +- 10 files changed, 110 insertions(+), 34 deletions(-) diff --git a/linkage-checker-exclusion-rules.xml b/linkage-checker-exclusion-rules.xml index 3544c6d48..1c4114b8c 100644 --- a/linkage-checker-exclusion-rules.xml +++ b/linkage-checker-exclusion-rules.xml @@ -19,6 +19,18 @@ Google Crypto Tink is an optional dependency of nimbus-jose-jwt + + + + + Not sure what is going on, the class is present. + + + + + + Not sure what is going on, the class is present. + diff --git a/pom.xml b/pom.xml index dbf156690..fc4a2ab81 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,7 @@ + 1.74 1.9.13 1.15 3.2.2 @@ -274,6 +275,16 @@ audience-annotations ${yetus.version} + + org.bouncycastle + bcpkix-jdk18on + ${bouncycastle.version} + + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.version} + org.codehaus.jackson jackson-core-asl @@ -449,6 +460,15 @@ org.apache.parquet parquet-hadoop + + + org.bouncycastle + bcpkix-jdk18on + + + org.bouncycastle + bcprov-jdk18on + org.slf4j slf4j-api @@ -531,7 +551,7 @@ com.google.cloud.tools linkage-checker-enforcer-rules - 1.5.12 + 1.5.13 org.codehaus.mojo @@ -758,6 +778,7 @@ The MIT License EDL 1.0 The Go license + Bouncy Castle Licence test,provided,system true @@ -929,6 +950,10 @@ com.nimbusds ${shadeBase}.com.nimbusds + + org.bouncycastle + ${shadeBase}.org.bouncycastle + net.jcip ${shadeBase}.net.jcip diff --git a/scripts/process_licenses.py b/scripts/process_licenses.py index a17c34843..bb43fbbf0 100644 --- a/scripts/process_licenses.py +++ b/scripts/process_licenses.py @@ -30,6 +30,7 @@ EDL_10_LICENSE = "EDL 1.0" MIT_LICENSE = "The MIT License" GO_LICENSE = "The Go license" +BOUNCY_CASTLE_LICENSE = "Bouncy Castle Licence " # The SDK does not need to include licenses of dependencies, which aren't shaded IGNORED_DEPENDENCIES = {"net.snowflake:snowflake-jdbc", "org.slf4j:slf4j-api"} @@ -57,6 +58,9 @@ "org.apache.parquet:parquet-format-structures": APACHE_LICENSE, "com.github.luben:zstd-jni": BSD_2_CLAUSE_LICENSE, "io.airlift:aircompressor": APACHE_LICENSE, + "org.bouncycastle:bcpkix-jdk18on": BOUNCY_CASTLE_LICENSE, + "org.bouncycastle:bcutil-jdk18on": BOUNCY_CASTLE_LICENSE, + "org.bouncycastle:bcprov-jdk18on": BOUNCY_CASTLE_LICENSE, } diff --git a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java index 97bdae0f1..b212d7244 100644 --- a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java +++ b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java @@ -40,7 +40,8 @@ public enum ErrorCode { MAKE_URI_FAILURE("0032"), OAUTH_REFRESH_TOKEN_ERROR("0033"), INVALID_CONFIG_PARAMETER("0034"), - MAX_BATCH_SIZE_EXCEEDED("0035"); + MAX_BATCH_SIZE_EXCEEDED("0035"), + CRYPTO_PROVIDER_ERROR("0036"); public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages"; diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java index ca39fed0d..0fa60bc8c 100644 --- a/src/main/java/net/snowflake/ingest/utils/Utils.java +++ b/src/main/java/net/snowflake/ingest/utils/Utils.java @@ -12,10 +12,12 @@ import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; import java.lang.management.MemoryUsage; +import java.lang.reflect.InvocationTargetException; import java.security.KeyFactory; import java.security.KeyPair; import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; +import java.security.Provider; import java.security.PublicKey; import java.security.Security; import java.security.interfaces.RSAPrivateCrtKey; @@ -26,20 +28,66 @@ import java.util.Map; import java.util.Properties; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.pkcs.PrivateKeyInfo; -import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider; -import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.PEMParser; -import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; -import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder; -import net.snowflake.client.jdbc.internal.org.bouncycastle.operator.InputDecryptorProvider; -import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo; import org.apache.commons.codec.binary.Base64; +import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; +import org.bouncycastle.openssl.PEMParser; +import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; +import org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder; +import org.bouncycastle.operator.InputDecryptorProvider; +import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo; /** Contains Ingest related utility functions */ public class Utils { private static final Logging logger = new Logging(Utils.class); + private static final String DEFAULT_SECURITY_PROVIDER_NAME = "org.bouncycastle.jce.provider.BouncyCastleProvider"; + + /** provider name */ + private static final String BOUNCY_CASTLE_PROVIDER = "BC"; + /** provider name for FIPS */ + private static final String BOUNCY_CASTLE_FIPS_PROVIDER = "BCFIPS"; + + static { + // Add Bouncy Castle to the security provider. This is required to + // verify the signature on OCSP response and attached certificates. + if (Security.getProvider(BOUNCY_CASTLE_PROVIDER) == null + && Security.getProvider(BOUNCY_CASTLE_FIPS_PROVIDER) == null) { + Security.addProvider(instantiateSecurityProvider()); + } + } + + public static Provider getProvider() { + final Provider bcProvider = Security.getProvider(BOUNCY_CASTLE_PROVIDER); + if (bcProvider != null) { + return bcProvider; + } + final Provider bcFipsProvider = Security.getProvider(BOUNCY_CASTLE_FIPS_PROVIDER); + if (bcFipsProvider != null) { + return bcFipsProvider; + } + throw new SFException(ErrorCode.INTERNAL_ERROR, "No security provider found"); + } + + private static Provider instantiateSecurityProvider() { + try { + logger.logInfo("Adding security provider {}", DEFAULT_SECURITY_PROVIDER_NAME); + System.out.println("Adding security provider {}"); + Class klass = Class.forName(DEFAULT_SECURITY_PROVIDER_NAME); + return (Provider) klass.getDeclaredConstructor().newInstance(); + } catch (ExceptionInInitializerError + | ClassNotFoundException + | NoSuchMethodException + | InstantiationException + | IllegalAccessException + | IllegalArgumentException + | InvocationTargetException + | SecurityException ex) { + throw new SFException(ErrorCode.CRYPTO_PROVIDER_ERROR, DEFAULT_SECURITY_PROVIDER_NAME, ex.getMessage()); + } + } + + /** * Assert when the String is null or Empty * @@ -183,7 +231,6 @@ public static PrivateKey parsePrivateKey(String key) { key = key.replaceAll("-+[A-Za-z ]+-+", ""); key = key.replaceAll("\\s", ""); - java.security.Security.addProvider(new BouncyCastleProvider()); byte[] encoded = Base64.decodeBase64(key); try { KeyFactory kf = KeyFactory.getInstance("RSA"); @@ -216,7 +263,6 @@ public static PrivateKey parseEncryptedPrivateKey(String key, String passphrase) } builder.append("\n-----END ENCRYPTED PRIVATE KEY-----"); key = builder.toString(); - Security.addProvider(new BouncyCastleProvider()); try { PEMParser pemParser = new PEMParser(new StringReader(key)); PKCS8EncryptedPrivateKeyInfo encryptedPrivateKeyInfo = @@ -225,7 +271,7 @@ public static PrivateKey parseEncryptedPrivateKey(String key, String passphrase) InputDecryptorProvider pkcs8Prov = new JceOpenSSLPKCS8DecryptorProviderBuilder().build(passphrase.toCharArray()); JcaPEMKeyConverter converter = - new JcaPEMKeyConverter().setProvider(BouncyCastleProvider.PROVIDER_NAME); + new JcaPEMKeyConverter().setProvider(Utils.getProvider().getName()); PrivateKeyInfo decryptedPrivateKeyInfo = encryptedPrivateKeyInfo.decryptPrivateKeyInfo(pkcs8Prov); return converter.getPrivateKey(decryptedPrivateKeyInfo); diff --git a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties index 9698799f7..268afd051 100644 --- a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties +++ b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties @@ -37,4 +37,5 @@ 0032=URI builder fail to build url: {0} 0033=OAuth token refresh failure: {0} 0034=Invalid config parameter: {0} -0035=Too large batch of rows passed to insertRows, the batch size cannot exceed {0} bytes, recommended batch size for optimal performance and memory utilization is {1} bytes. We recommend splitting large batches into multiple smaller ones and call insertRows for each smaller batch separately. \ No newline at end of file +0035=Too large batch of rows passed to insertRows, the batch size cannot exceed {0} bytes, recommended batch size for optimal performance and memory utilization is {1} bytes. We recommend splitting large batches into multiple smaller ones and call insertRows for each smaller batch separately. +0036=Failed to load {0}. If you use FIPS, import BouncyCastleFipsProvider in the application: {1} \ No newline at end of file diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java index b2ebe1bdd..ba14ff610 100644 --- a/src/test/java/net/snowflake/ingest/TestUtils.java +++ b/src/test/java/net/snowflake/ingest/TestUtils.java @@ -41,7 +41,6 @@ import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; -import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider; import net.snowflake.ingest.streaming.InsertValidationResponse; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel; import net.snowflake.ingest.utils.Constants; @@ -123,8 +122,6 @@ private static void init() throws Exception { role = Optional.ofNullable(profile.get(ROLE)).map(r -> r.asText()).orElse("DEFAULT_ROLE"); privateKeyPem = profile.get(PRIVATE_KEY).asText(); - java.security.Security.addProvider(new BouncyCastleProvider()); - byte[] encoded = Base64.decodeBase64(privateKeyPem); KeyFactory kf = KeyFactory.getInstance("RSA"); diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java index 0b8e8b4cd..5a77a51db 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java @@ -39,7 +39,6 @@ import javax.crypto.BadPaddingException; import javax.crypto.IllegalBlockSizeException; import javax.crypto.NoSuchPaddingException; -import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider; import net.snowflake.ingest.streaming.OpenChannelRequest; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.Cryptor; @@ -47,7 +46,6 @@ import net.snowflake.ingest.utils.ParameterProvider; import net.snowflake.ingest.utils.SFException; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; @@ -277,11 +275,6 @@ TestContext>> create() { TestContextFactory testContextFactory; - @Before - public void setup() { - java.security.Security.addProvider(new BouncyCastleProvider()); - } - private SnowflakeStreamingIngestChannelInternal addChannel1(TestContext testContext) { return testContext .channelBuilder("channel1") diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java index ec9f671bb..17f8844c9 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java @@ -19,7 +19,6 @@ import java.io.StringWriter; import java.security.KeyPair; import java.security.PrivateKey; -import java.security.Security; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collections; @@ -40,13 +39,6 @@ import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; import net.snowflake.client.jdbc.internal.google.common.collect.Sets; -import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.nist.NISTObjectIdentifiers; -import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider; -import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JcaPEMWriter; -import net.snowflake.client.jdbc.internal.org.bouncycastle.operator.OperatorCreationException; -import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfoBuilder; -import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.jcajce.JcaPKCS8EncryptedPrivateKeyInfoBuilder; -import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.jcajce.JcePKCSPBEOutputEncryptorBuilder; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.OpenChannelRequest; @@ -59,6 +51,12 @@ import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; +import org.bouncycastle.asn1.nist.NISTObjectIdentifiers; +import org.bouncycastle.openssl.jcajce.JcaPEMWriter; +import org.bouncycastle.operator.OperatorCreationException; +import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfoBuilder; +import org.bouncycastle.pkcs.jcajce.JcaPKCS8EncryptedPrivateKeyInfoBuilder; +import org.bouncycastle.pkcs.jcajce.JcePKCSPBEOutputEncryptorBuilder; import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; @@ -293,7 +291,6 @@ public void testEncryptedPrivateKey() throws Exception { private String generateAESKey(PrivateKey key, char[] passwd) throws IOException, OperatorCreationException { - Security.addProvider(new BouncyCastleProvider()); StringWriter writer = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(writer); PKCS8EncryptedPrivateKeyInfoBuilder pkcs8EncryptedPrivateKeyInfoBuilder = @@ -301,7 +298,7 @@ private String generateAESKey(PrivateKey key, char[] passwd) pemWriter.writeObject( pkcs8EncryptedPrivateKeyInfoBuilder.build( new JcePKCSPBEOutputEncryptorBuilder(NISTObjectIdentifiers.id_aes256_CBC) - .setProvider(BouncyCastleProvider.PROVIDER_NAME) + .setProvider(Utils.getProvider().getName()) .build(passwd))); pemWriter.close(); return writer.toString(); diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java index d9e6e3ddd..2a5023a3a 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java @@ -1,6 +1,6 @@ package net.snowflake.ingest.streaming.internal.datatypes; -import net.snowflake.client.jdbc.internal.org.bouncycastle.util.encoders.Hex; +import org.bouncycastle.util.encoders.Hex; import org.junit.Test; public class BinaryIT extends AbstractDataTypeTest {