Merge branch 'master' into SNOW-1213115-Move-test-categories-to-have-…

…equal-distribution
snowflakedb · Jul 18, 2024 · 6af9d36 · 6af9d36
2 parents f21e521 + 74a37b2
commit 6af9d36
Show file tree

Hide file tree

Showing 257 changed files with 19,518 additions and 12,485 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -7,6 +7,7 @@ SNOW-XXXXX
 - [ ] The code is correctly formatted (run `mvn -P check-style validate`)
 - [ ] New public API is not unnecessary exposed (run `mvn verify` and inspect `target/japicmp/japicmp.html`)
 - [ ] The pull request name is prefixed with `SNOW-XXXX: `
+- [ ] Code is in compliance with internal logging requirements
 
 ## External contributors - please answer these questions before submitting a pull request. Thanks!
 

diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -36,16 +36,78 @@ jobs:
                 WHITESOURCE_API_KEY: ${{ secrets.WHITESOURCE_API_KEY }}
               run: ./ci/build.sh
 
+    test-windows:
+        needs: build
+        name: ${{ matrix.runConfig.cloud }} Windows java ${{ matrix.runConfig.javaVersion }} JDBC${{ matrix.additionalMavenProfile }} ${{ matrix.category }}
+        runs-on: windows-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                runConfig: [ {cloud: 'AWS', javaVersion: '8'}, {cloud: 'GCP', javaVersion: '11'}, {cloud: 'AZURE', javaVersion: '17'}, {cloud: 'AWS', javaVersion: '21'}]
+                category: ['TestCategoryResultSet,TestCategoryOthers,TestCategoryLoader,TestCategoryDiagnostic', 'TestCategoryConnection,TestCategoryStatement', 'TestCategoryArrow,TestCategoryCore', 'TestCategoryFips']
+                additionalMavenProfile: ['']
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-java@v4
+              with:
+                java-version: ${{ matrix.runConfig.javaVersion }}
+                distribution: 'temurin'
+                cache: maven
+            - uses: actions/setup-python@v4
+              with:
+                python-version: '3.7'
+                architecture: 'x64'
+            - name: Tests
+              shell: cmd
+              env:
+                PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
+                CLOUD_PROVIDER: ${{ matrix.runConfig.cloud }}
+                JDBC_TEST_CATEGORY: ${{ matrix.category }}
+                ADDITIONAL_MAVEN_PROFILE: ${{ matrix.additionalMavenProfile }}
+              run: ci\\test_windows.bat
+
+    test-mac:
+        needs: build
+        name: ${{ matrix.runConfig.cloud }} Mac java ${{ matrix.runConfig.javaVersion }} JDBC${{ matrix.additionalMavenProfile }} ${{ matrix.category }}
+        runs-on: macos-13
+        strategy:
+            fail-fast: false
+            matrix:
+                runConfig: [ {cloud: 'AWS', javaVersion: '8'}, {cloud: 'GCP', javaVersion: '11'}, {cloud: 'AZURE', javaVersion: '17'}, {cloud: 'AWS', javaVersion: '21'}]
+                category: ['TestCategoryResultSet,TestCategoryOthers,TestCategoryLoader,TestCategoryDiagnostic', 'TestCategoryConnection,TestCategoryStatement', 'TestCategoryArrow,TestCategoryCore', 'TestCategoryFips']
+                additionalMavenProfile: ['']
+        steps:
+            - uses: actions/checkout@v4
+            - uses: actions/setup-java@v4
+              with:
+                java-version: ${{ matrix.runConfig.javaVersion }}
+                distribution: 'temurin'
+                cache: maven
+            - uses: actions/setup-python@v4
+              with:
+                python-version: '3.7'
+            - name: Install Homebrew Bash
+              shell: bash
+              run: brew install bash
+            - name: Tests
+              shell: bash
+              env:
+                PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
+                CLOUD_PROVIDER: ${{ matrix.runConfig.cloud }}
+                JDBC_TEST_CATEGORY: ${{ matrix.category }}
+                ADDITIONAL_MAVEN_PROFILE: ${{ matrix.additionalMavenProfile }}
+              run: /usr/local/bin/bash ./ci/test_mac.sh
+
     test-linux:
         needs: build
-        name: ${{ matrix.cloud }} JDBC${{ matrix.additionalMavenProfile }} ${{ matrix.category }} on ${{ matrix.image }}
+        name: ${{ matrix.cloud }} Linux java on ${{ matrix.image }} JDBC${{ matrix.additionalMavenProfile }} ${{ matrix.category }}
         runs-on: ubuntu-latest
         strategy:
             fail-fast: false
             matrix:
-                image: [ 'jdbc-centos7-openjdk8', 'jdbc-centos7-openjdk11', 'jdbc-centos7-openjdk17' ]
-                cloud: [ 'AWS' ]
-                category: ['TestCategoryResultSet,TestCategoryStatement', 'TestCategoryOthers,TestCategoryCore', 'TestCategoryArrow,TestCategoryLoader,TestCategoryConnection', 'TestCategoryFips']
+                image: [ 'jdbc-centos7-openjdk8', 'jdbc-centos7-openjdk11', 'jdbc-centos7-openjdk17', 'jdbc-centos7-openjdk21' ]
+                cloud: [ 'AWS', 'AZURE', 'GCP' ]
+                category: ['TestCategoryResultSet,TestCategoryStatement,TestCategoryLoader', 'TestCategoryOthers', 'TestCategoryArrow,TestCategoryConnection,TestCategoryCore,TestCategoryDiagnostic', 'TestCategoryFips']
                 additionalMavenProfile: ['', '-Dthin-jar']
         steps:
             - uses: actions/checkout@v1

diff --git a/.github/workflows/parameters_azure.json.gpg b/.github/workflows/parameters_azure.json.gpg
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,3 +1,15 @@
+**JDBC Driver 3.17.0**
+
+- \||Please Refer to Release Notes at https://docs.snowflake.com/en/release-notes/clients-drivers/jdbc
+
+**JDBC Driver 3.16.1**
+
+- \||Please Refer to Release Notes at https://docs.snowflake.com/en/release-notes/clients-drivers/jdbc
+
+**JDBC Driver 3.16.0**
+
+- \||Please Refer to Release Notes at https://docs.snowflake.com/en/release-notes/clients-drivers/jdbc
+
 **JDBC Driver 3.15.1**
 
 - \||Please Refer to Release Notes at https://docs.snowflake.com/en/release-notes/clients-drivers/jdbc

diff --git a/FIPS/pom.xml b/FIPS/pom.xml
@@ -5,12 +5,12 @@
   <parent>
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-jdbc-parent</artifactId>
-    <version>3.15.2-SNAPSHOT</version>
+    <version>3.17.1-SNAPSHOT</version>
     <relativePath>../parent-pom.xml</relativePath>
   </parent>
 
   <artifactId>snowflake-jdbc-fips</artifactId>
-  <version>3.15.2-SNAPSHOT</version>
+  <version>3.17.1-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <name>snowflake-jdbc-fips</name>
@@ -323,23 +323,6 @@
           </execution>
         </executions>
       </plugin>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>exec-maven-plugin</artifactId>
-        <version>${version.plugin.exec}</version>
-        <executions>
-          <execution>
-            <id>check-shaded-content</id>
-            <phase>verify</phase>
-            <goals>
-              <goal>exec</goal>
-            </goals>
-            <configuration>
-              <executable>${basedir}/scripts/check_content.sh</executable>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
   <reporting>
@@ -466,13 +449,6 @@
                        <pattern>org.objectweb</pattern>
                        <shadedPattern>${shadeBase}.org.objectweb</shadedPattern>
                      </relocation>
-                     <relocation>
-                       <pattern>com.sun</pattern>
-                       <shadedPattern>${shadeBase}.com.sun</shadedPattern>
-                       <excludes>
-                         <exclude>com.sun.jna.**</exclude>
-                       </excludes>
-                     </relocation>
                      <relocation>
                        <pattern>io.netty</pattern>
                        <shadedPattern>${shadeBase}.io.netty</shadedPattern>
@@ -669,6 +645,35 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>check-content</id>
+      <activation>
+        <os>
+          <family>!windows</family>
+        </os>
+      </activation>
+      <build>
+        <plugins>      
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <version>${version.plugin.exec}</version>
+            <executions>
+              <execution>
+                <id>check-shaded-content</id>
+                <phase>verify</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <executable>${basedir}/scripts/check_content.sh</executable>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>java-9</id>
       <activation>

diff --git a/FIPS/src/test/java/net/snowflake/client/RunningOnGCP.java b/FIPS/src/test/java/net/snowflake/client/RunningOnGCP.java
@@ -0,0 +1,12 @@
+/*
+ * Copyright (c) 2012-2024 Snowflake Computing Inc. All right reserved.
+ */
+package net.snowflake.client;
+
+/** Run tests only on specified cloud provider or ignore */
+public class RunningOnGCP implements ConditionalIgnoreRule.IgnoreCondition {
+  public boolean isSatisfied() {
+    String cloudProvider = TestUtil.systemGetEnv("CLOUD_PROVIDER");
+    return cloudProvider != null && cloudProvider.equalsIgnoreCase("GCP");
+  }
+}
diff --git a/FIPS/src/test/java/net/snowflake/client/RunningOnWinMac.java b/FIPS/src/test/java/net/snowflake/client/RunningOnWinMac.java
diff --git a/FIPS/src/test/java/net/snowflake/client/TestUtil.java b/FIPS/src/test/java/net/snowflake/client/TestUtil.java
@@ -37,14 +37,15 @@ public interface TestRunInterface {
   /**
    * System.getenv wrapper. If System.getenv raises an SecurityException, it is ignored and returns
    * null.
-   *
+   * @deprecated This method should be replaced by SnowflakeUtil.systemGetEnv.
    * <p>This is replicated from SnowflakeUtil.systemGetEnv, because the old driver doesn't have that
    * function for the tests to use it. Replace this function call with SnowflakeUtil.systemGetEnv
    * when it is available.
    *
    * @param env the environment variable name.
    * @return the environment variable value if set, otherwise null.
    */
+  @Deprecated
   public static String systemGetEnv(String env) {
     try {
       return System.getenv(env);

diff --git a/FIPS/src/test/java/net/snowflake/client/jdbc/ConnectionFipsIT.java b/FIPS/src/test/java/net/snowflake/client/jdbc/ConnectionFipsIT.java
@@ -21,6 +21,7 @@
 import javax.net.ssl.HttpsURLConnection;
 import net.snowflake.client.AbstractDriverIT;
 import net.snowflake.client.ConditionalIgnoreRule;
+import net.snowflake.client.RunningOnGCP;
 import net.snowflake.client.RunningOnGithubActions;
 import net.snowflake.client.category.TestCategoryFips;
 import net.snowflake.client.core.SecurityUtil;
@@ -289,7 +290,12 @@ public void testConnectUsingKeyPair() throws Exception {
     DriverManager.getConnection(uri, properties).close();
   }
 
+  /**
+   * Test case for connecting with FIPS and executing a query.
+   * Currently ignored execution on GCP due to exception thrown "SSlException Could not generate XDH keypair"
+   */
   @Test
+  @ConditionalIgnoreRule.ConditionalIgnore(condition = RunningOnGCP.class)
   public void connectWithFipsAndQuery() throws SQLException {
     try (Connection con = getConnection()) {
       Statement statement = con.createStatement();

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -56,7 +56,7 @@ timestamps {
       e.printStackTrace()
     }
 
-    jdkToParams = ['openjdk8': 'jdbc-centos7-openjdk8', 'openjdk11': 'jdbc-centos7-openjdk11', 'openjdk17': 'jdbc-centos7-openjdk17'].collectEntries { jdk, image ->
+    jdkToParams = ['openjdk8': 'jdbc-centos7-openjdk8', 'openjdk11': 'jdbc-centos7-openjdk11', 'openjdk17': 'jdbc-centos7-openjdk17', 'openjdk21': 'jdbc-centos7-openjdk21'].collectEntries { jdk, image ->
       return [(jdk): [
         string(name: 'client_git_branch', value: scmInfo.GIT_BRANCH),
         string(name: 'client_git_commit', value: scmInfo.GIT_COMMIT),

diff --git a/ci/_init.sh b/ci/_init.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/local/bin/env bash
 set -e
 
 export PLATFORM=$(echo $(uname) | tr '[:upper:]' '[:lower:]')
@@ -23,18 +23,21 @@ declare -A TEST_IMAGE_NAMES=(
     [$DRIVER_NAME-centos7-openjdk8]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-openjdk8-test:$TEST_IMAGE_VERSION
     [$DRIVER_NAME-centos7-openjdk11]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-openjdk11-test:$TEST_IMAGE_VERSION
     [$DRIVER_NAME-centos7-openjdk17]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-openjdk17-test:$TEST_IMAGE_VERSION
+    [$DRIVER_NAME-centos7-openjdk21]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-openjdk21-test:$TEST_IMAGE_VERSION
 )
 export TEST_IMAGE_NAMES
 
 declare -A TEST_IMAGE_DOCKERFILES=(
     [$DRIVER_NAME-centos7-openjdk8]=jdbc-centos7-openjdk-test
     [$DRIVER_NAME-centos7-openjdk11]=jdbc-centos7-openjdk-test
     [$DRIVER_NAME-centos7-openjdk17]=jdbc-centos7-openjdk-test
+    [$DRIVER_NAME-centos7-openjdk21]=jdbc-centos7-openjdk-test
 )
 
 declare -A TEST_IMAGE_BUILD_ARGS=(
     [$DRIVER_NAME-centos7-openjdk8]="--target jdbc-centos7-openjdk-yum --build-arg=JDK_PACKAGE=java-1.8.0-openjdk-devel"
     [$DRIVER_NAME-centos7-openjdk11]="--target jdbc-centos7-openjdk-yum --build-arg=JDK_PACKAGE=java-11-openjdk-devel" # pragma: allowlist secret
     [$DRIVER_NAME-centos7-openjdk17]="--target jdbc-centos7-openjdk17"
+    [$DRIVER_NAME-centos7-openjdk21]="--target jdbc-centos7-openjdk21"
 )
 
diff --git a/ci/container/test_component.sh b/ci/container/test_component.sh
@@ -1,11 +1,12 @@
 #!/bin/bash -e
 #
-# Test JDBC for Linux
+# Test JDBC for Linux/MAC
 #
 set -o pipefail
 THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 export WORKSPACE=${WORKSPACE:-/mnt/workspace}
 export SOURCE_ROOT=${SOURCE_ROOT:-/mnt/host}
+MVNW_EXE=$SOURCE_ROOT/mvnw
 
 echo "[INFO] Download JDBC Integration test cases and libraries"
 source $THIS_DIR/download_artifact.sh
@@ -76,15 +77,15 @@ export MAVEN_OPTS="$MAVEN_OPTS -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=fa
 cd $SOURCE_ROOT
 
 # Avoid connection timeout on plugin dependency fetch or fail-fast when dependency cannot be fetched
-mvn --batch-mode --show-version dependency:resolve-plugins
+$MVNW_EXE --batch-mode --show-version dependency:go-offline
 
 for c in "${CATEGORY[@]}"; do
     c=$(echo $c | sed 's/ *$//g')
     if [[ "$is_old_driver" == "true" ]]; then
         pushd TestOnly >& /dev/null
-            JDBC_VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version --batch-mode | grep -v "[INFO]")
+            JDBC_VERSION=$($MVNW_EXE org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version --batch-mode | grep -v "[INFO]")
             echo "[INFO] Run JDBC $JDBC_VERSION tests"
-            mvn -DjenkinsIT \
+            $MVNW_EXE -DjenkinsIT \
                 -Djava.io.tmpdir=$WORKSPACE \
                 -Djacoco.skip.instrument=false \
                 -DtestCategory=net.snowflake.client.category.$c \
@@ -95,7 +96,7 @@ for c in "${CATEGORY[@]}"; do
     elif [[ "$c" == "TestCategoryFips" ]]; then
         pushd FIPS >& /dev/null
             echo "[INFO] Run Fips tests"
-            mvn -DjenkinsIT \
+            $MVNW_EXE -DjenkinsIT \
                 -Djava.io.tmpdir=$WORKSPACE \
                 -Djacoco.skip.instrument=false \
                 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
@@ -105,7 +106,7 @@ for c in "${CATEGORY[@]}"; do
         popd >& /dev/null
     else
         echo "[INFO] Run $c tests"
-        mvn -DjenkinsIT \
+        $MVNW_EXE -DjenkinsIT \
             -Djava.io.tmpdir=$WORKSPACE \
             -Djacoco.skip.instrument=false \
             -DtestCategory=net.snowflake.client.category.$c \

diff --git a/ci/image/Dockerfile.jdbc-centos7-openjdk-test b/ci/image/Dockerfile.jdbc-centos7-openjdk-test
@@ -88,4 +88,22 @@ RUN export JAVA_HOME=/opt/jdk-17 && \
         -Dnot-self-contained-jar \
         --batch-mode --fail-never compile && \
     mv $HOME/.m2 /home/user && \
-    chmod -R 777 /home/user/.m2
+    chmod -R 777 /home/user/.m2
+
+###### OpenJDK 21 from archive (not available in yum)
+FROM jdbc-centos7-openjdk-base AS jdbc-centos7-openjdk21
+
+# Java
+RUN curl -o - https://download.java.net/java/GA/jdk21.0.2/f2283984656d49d69e91c558476027ac/13/GPL/openjdk-21.0.2_linux-x64_bin.tar.gz | tar xfz - -C /opt && \
+    ln -s /opt/jdk-21.0.2 /opt/jdk-21
+
+RUN sed -i /usr/local/bin/entrypoint.sh -e '/^exec/i export JAVA_HOME=/opt/jdk-21'
+RUN sed -i /usr/local/bin/entrypoint.sh -e '/^exec/i export PATH=$JAVA_HOME/bin:$PATH'
+
+RUN export JAVA_HOME=/opt/jdk-21 && \
+    cd /root && \
+    mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
+        -Dnot-self-contained-jar \
+        --batch-mode --fail-never compile && \
+    mv $HOME/.m2 /home/user && \
+    chmod -R 777 /home/user/.m2