SNOW-926149 Fix issues while using snowflake-jdbc-fips

linkage-checker-exclusion-rules.xml

@@ -19,6 +19,18 @@
         <Reason>Google Crypto Tink is an optional dependency of nimbus-jose-jwt</Reason>
     </LinkageError>
 
+    <LinkageError>
+        <Target>
+            <Class name="org.bouncycastle.asn1.x509.qualified.QCStatement" />
+        </Target>
+        <Reason>Not sure what is going on, the class is present.</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Class name="org.bouncycastle.asn1.x9.X9FieldID" />
+        </Target>
+        <Reason>Not sure what is going on, the class is present.</Reason>
+    </LinkageError>
     <!-- Hadoop-related linkage errors-->
     <LinkageError>
         <Source>

pom.xml

@@ -35,6 +35,7 @@
 
   <!-- Set our Language Level to Java 8 -->
   <properties>
+    <bouncycastle.version>1.74</bouncycastle.version>
     <codehaus.version>1.9.13</codehaus.version>
     <commonscodec.version>1.15</commonscodec.version>
     <commonscollections.version>3.2.2</commonscollections.version>
@@ -274,6 +275,16 @@
         <artifactId>audience-annotations</artifactId>
         <version>${yetus.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.bouncycastle</groupId>
+        <artifactId>bcpkix-jdk18on</artifactId>
+        <version>${bouncycastle.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.bouncycastle</groupId>
+        <artifactId>bcprov-jdk18on</artifactId>
+        <version>${bouncycastle.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-core-asl</artifactId>
@@ -449,6 +460,15 @@
       <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-hadoop</artifactId>
     </dependency>
+
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcpkix-jdk18on</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcprov-jdk18on</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
@@ -531,7 +551,7 @@
             <dependency>
               <groupId>com.google.cloud.tools</groupId>
               <artifactId>linkage-checker-enforcer-rules</artifactId>
-              <version>1.5.12</version>
+              <version>1.5.13</version>
             </dependency>
             <dependency>
               <groupId>org.codehaus.mojo</groupId>
@@ -758,6 +778,7 @@
             <includedLicense>The MIT License</includedLicense>
             <includedLicense>EDL 1.0</includedLicense>
             <includedLicense>The Go license</includedLicense>
+            <includedLicense>Bouncy Castle Licence</includedLicense>
           </includedLicenses>
           <excludedScopes>test,provided,system</excludedScopes>
           <failOnBlacklist>true</failOnBlacklist>
@@ -929,6 +950,10 @@
                   <pattern>com.nimbusds</pattern>
                   <shadedPattern>${shadeBase}.com.nimbusds</shadedPattern>
                 </relocation>
+                <relocation>
+                  <pattern>org.bouncycastle</pattern>
+                  <shadedPattern>${shadeBase}.org.bouncycastle</shadedPattern>
+                </relocation>
                 <relocation>
                   <pattern>net.jcip</pattern>
                   <shadedPattern>${shadeBase}.net.jcip</shadedPattern>

scripts/process_licenses.py

@@ -30,6 +30,7 @@
 EDL_10_LICENSE = "EDL 1.0"
 MIT_LICENSE = "The MIT License"
 GO_LICENSE = "The Go license"
+BOUNCY_CASTLE_LICENSE = "Bouncy Castle Licence <https://www.bouncycastle.org/licence.html>"
 
 # The SDK does not need to include licenses of dependencies, which aren't shaded
 IGNORED_DEPENDENCIES = {"net.snowflake:snowflake-jdbc", "org.slf4j:slf4j-api"}
@@ -57,6 +58,9 @@
     "org.apache.parquet:parquet-format-structures": APACHE_LICENSE,
     "com.github.luben:zstd-jni": BSD_2_CLAUSE_LICENSE,
     "io.airlift:aircompressor": APACHE_LICENSE,
+    "org.bouncycastle:bcpkix-jdk18on": BOUNCY_CASTLE_LICENSE,
+    "org.bouncycastle:bcutil-jdk18on": BOUNCY_CASTLE_LICENSE,
+    "org.bouncycastle:bcprov-jdk18on": BOUNCY_CASTLE_LICENSE,
 }
 
 

src/main/java/net/snowflake/ingest/utils/ErrorCode.java

@@ -40,7 +40,8 @@ public enum ErrorCode {
   MAKE_URI_FAILURE("0032"),
   OAUTH_REFRESH_TOKEN_ERROR("0033"),
   INVALID_CONFIG_PARAMETER("0034"),
-  MAX_BATCH_SIZE_EXCEEDED("0035");
+  MAX_BATCH_SIZE_EXCEEDED("0035"),
+  CRYPTO_PROVIDER_ERROR("0036");
 
   public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages";
 

src/main/java/net/snowflake/ingest/utils/Utils.java

@@ -12,10 +12,12 @@
 import java.lang.management.BufferPoolMXBean;
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryUsage;
+import java.lang.reflect.InvocationTargetException;
 import java.security.KeyFactory;
 import java.security.KeyPair;
 import java.security.NoSuchAlgorithmException;
 import java.security.PrivateKey;
+import java.security.Provider;
 import java.security.PublicKey;
 import java.security.Security;
 import java.security.interfaces.RSAPrivateCrtKey;
@@ -26,20 +28,66 @@
 import java.util.Map;
 import java.util.Properties;
 import net.snowflake.client.core.SFSessionProperty;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.PEMParser;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.operator.InputDecryptorProvider;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo;
 import org.apache.commons.codec.binary.Base64;
+import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
+import org.bouncycastle.openssl.PEMParser;
+import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;
+import org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder;
+import org.bouncycastle.operator.InputDecryptorProvider;
+import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo;
 
 /** Contains Ingest related utility functions */
 public class Utils {
 
   private static final Logging logger = new Logging(Utils.class);
 
+  private static final String DEFAULT_SECURITY_PROVIDER_NAME = "org.bouncycastle.jce.provider.BouncyCastleProvider";
+
+  /** provider name */
+  private static final String BOUNCY_CASTLE_PROVIDER = "BC";
+  /** provider name for FIPS */
+  private static final String BOUNCY_CASTLE_FIPS_PROVIDER = "BCFIPS";
+
+  static {
+    // Add Bouncy Castle to the security provider. This is required to
+    // verify the signature on OCSP response and attached certificates.
+    if (Security.getProvider(BOUNCY_CASTLE_PROVIDER) == null
+            && Security.getProvider(BOUNCY_CASTLE_FIPS_PROVIDER) == null) {
+      Security.addProvider(instantiateSecurityProvider());
+    }
+  }
+
+  public static Provider getProvider() {
+    final Provider bcProvider = Security.getProvider(BOUNCY_CASTLE_PROVIDER);
+    if (bcProvider != null) {
+      return bcProvider;
+    }
+    final Provider bcFipsProvider = Security.getProvider(BOUNCY_CASTLE_FIPS_PROVIDER);
+    if (bcFipsProvider != null) {
+      return bcFipsProvider;
+    }
+    throw new SFException(ErrorCode.INTERNAL_ERROR, "No security provider found");
+  }
+
+  private static Provider instantiateSecurityProvider() {
+    try {
+      logger.logInfo("Adding security provider {}", DEFAULT_SECURITY_PROVIDER_NAME);
+      System.out.println("Adding security provider {}");
+      Class klass = Class.forName(DEFAULT_SECURITY_PROVIDER_NAME);
+      return (Provider) klass.getDeclaredConstructor().newInstance();
+    } catch (ExceptionInInitializerError
+             | ClassNotFoundException
+             | NoSuchMethodException
+             | InstantiationException
+             | IllegalAccessException
+             | IllegalArgumentException
+             | InvocationTargetException
+             | SecurityException ex) {
+      throw new SFException(ErrorCode.CRYPTO_PROVIDER_ERROR, DEFAULT_SECURITY_PROVIDER_NAME, ex.getMessage());
+    }
+  }
+
+
   /**
    * Assert when the String is null or Empty
    *
@@ -183,7 +231,6 @@ public static PrivateKey parsePrivateKey(String key) {
     key = key.replaceAll("-+[A-Za-z ]+-+", "");
     key = key.replaceAll("\\s", "");
 
-    java.security.Security.addProvider(new BouncyCastleProvider());
     byte[] encoded = Base64.decodeBase64(key);
     try {
       KeyFactory kf = KeyFactory.getInstance("RSA");
@@ -216,7 +263,6 @@ public static PrivateKey parseEncryptedPrivateKey(String key, String passphrase)
     }
     builder.append("\n-----END ENCRYPTED PRIVATE KEY-----");
     key = builder.toString();
-    Security.addProvider(new BouncyCastleProvider());
     try {
       PEMParser pemParser = new PEMParser(new StringReader(key));
       PKCS8EncryptedPrivateKeyInfo encryptedPrivateKeyInfo =
@@ -225,7 +271,7 @@ public static PrivateKey parseEncryptedPrivateKey(String key, String passphrase)
       InputDecryptorProvider pkcs8Prov =
           new JceOpenSSLPKCS8DecryptorProviderBuilder().build(passphrase.toCharArray());
       JcaPEMKeyConverter converter =
-          new JcaPEMKeyConverter().setProvider(BouncyCastleProvider.PROVIDER_NAME);
+          new JcaPEMKeyConverter().setProvider(Utils.getProvider().getName());
       PrivateKeyInfo decryptedPrivateKeyInfo =
           encryptedPrivateKeyInfo.decryptPrivateKeyInfo(pkcs8Prov);
       return converter.getPrivateKey(decryptedPrivateKeyInfo);

src/main/resources/net/snowflake/ingest/ingest_error_messages.properties

@@ -37,4 +37,5 @@
 0032=URI builder fail to build url: {0}
 0033=OAuth token refresh failure: {0}
 0034=Invalid config parameter: {0}
-0035=Too large batch of rows passed to insertRows, the batch size cannot exceed {0} bytes, recommended batch size for optimal performance and memory utilization is {1} bytes. We recommend splitting large batches into multiple smaller ones and call insertRows for each smaller batch separately.
+0035=Too large batch of rows passed to insertRows, the batch size cannot exceed {0} bytes, recommended batch size for optimal performance and memory utilization is {1} bytes. We recommend splitting large batches into multiple smaller ones and call insertRows for each smaller batch separately.
+0036=Failed to load {0}. If you use FIPS, import BouncyCastleFipsProvider in the application: {1}

src/test/java/net/snowflake/ingest/TestUtils.java

@@ -41,7 +41,6 @@
 import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.utils.Constants;
@@ -123,8 +122,6 @@ private static void init() throws Exception {
       role = Optional.ofNullable(profile.get(ROLE)).map(r -> r.asText()).orElse("DEFAULT_ROLE");
       privateKeyPem = profile.get(PRIVATE_KEY).asText();
 
-      java.security.Security.addProvider(new BouncyCastleProvider());
-
       byte[] encoded = Base64.decodeBase64(privateKeyPem);
       KeyFactory kf = KeyFactory.getInstance("RSA");
 

src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java

@@ -39,15 +39,13 @@
 import javax.crypto.BadPaddingException;
 import javax.crypto.IllegalBlockSizeException;
 import javax.crypto.NoSuchPaddingException;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Cryptor;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
@@ -277,11 +275,6 @@ TestContext<List<List<Object>>> create() {
 
   TestContextFactory<?> testContextFactory;
 
-  @Before
-  public void setup() {
-    java.security.Security.addProvider(new BouncyCastleProvider());
-  }
-
   private SnowflakeStreamingIngestChannelInternal<?> addChannel1(TestContext<?> testContext) {
     return testContext
         .channelBuilder("channel1")

src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java

@@ -19,7 +19,6 @@
 import java.io.StringWriter;
 import java.security.KeyPair;
 import java.security.PrivateKey;
-import java.security.Security;
 import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -40,13 +39,6 @@
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost;
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.nist.NISTObjectIdentifiers;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JcaPEMWriter;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.operator.OperatorCreationException;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfoBuilder;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.jcajce.JcaPKCS8EncryptedPrivateKeyInfoBuilder;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.jcajce.JcePKCSPBEOutputEncryptorBuilder;
 import net.snowflake.ingest.TestUtils;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
@@ -59,6 +51,12 @@
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
 import net.snowflake.ingest.utils.Utils;
+import org.bouncycastle.asn1.nist.NISTObjectIdentifiers;
+import org.bouncycastle.openssl.jcajce.JcaPEMWriter;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfoBuilder;
+import org.bouncycastle.pkcs.jcajce.JcaPKCS8EncryptedPrivateKeyInfoBuilder;
+import org.bouncycastle.pkcs.jcajce.JcePKCSPBEOutputEncryptorBuilder;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -293,15 +291,14 @@ public void testEncryptedPrivateKey() throws Exception {
 
   private String generateAESKey(PrivateKey key, char[] passwd)
       throws IOException, OperatorCreationException {
-    Security.addProvider(new BouncyCastleProvider());
     StringWriter writer = new StringWriter();
     JcaPEMWriter pemWriter = new JcaPEMWriter(writer);
     PKCS8EncryptedPrivateKeyInfoBuilder pkcs8EncryptedPrivateKeyInfoBuilder =
         new JcaPKCS8EncryptedPrivateKeyInfoBuilder(key);
     pemWriter.writeObject(
         pkcs8EncryptedPrivateKeyInfoBuilder.build(
             new JcePKCSPBEOutputEncryptorBuilder(NISTObjectIdentifiers.id_aes256_CBC)
-                .setProvider(BouncyCastleProvider.PROVIDER_NAME)
+                    .setProvider(Utils.getProvider().getName())
                 .build(passwd)));
     pemWriter.close();
     return writer.toString();

src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java

@@ -1,6 +1,6 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
-import net.snowflake.client.jdbc.internal.org.bouncycastle.util.encoders.Hex;
+import org.bouncycastle.util.encoders.Hex;
 import org.junit.Test;
 
 public class BinaryIT extends AbstractDataTypeTest {