From 4625079b61ed4feaafa6fab71e8339ddbfee8dbf Mon Sep 17 00:00:00 2001 From: Piotr Fus Date: Wed, 15 Nov 2023 10:05:46 +0100 Subject: [PATCH] SNOW-937199 Build on Java 11 and 17 on Github actions --- .github/workflows/build-test.yml | 4 +- FIPS/pom.xml | 40 ++++++++ ci/_init.sh | 28 ++++-- ci/build.sh | 4 +- ci/image/.gitignore | 1 + ci/image/Dockerfile.jdbc-centos7-openjdk-test | 91 +++++++++++++++++++ ci/image/build.sh | 20 ++-- ci/image/update.sh | 15 +-- pom.xml | 10 +- .../client/core/SessionUtilLatestIT.java | 4 +- .../snowflake/client/core/StmtUtilTest.java | 2 +- .../client/jdbc/ConnectionWithOCSPModeIT.java | 3 + 12 files changed, 179 insertions(+), 43 deletions(-) create mode 100644 ci/image/Dockerfile.jdbc-centos7-openjdk-test diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 321e00b01..92a7a8a54 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -35,8 +35,9 @@ jobs: name: ${{ matrix.cloud }} JDBC ${{ matrix.category }} on ${{ matrix.image }} runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - image: [ 'jdbc-centos6-default' ] + image: [ 'jdbc-centos6-default', 'jdbc-centos7-openjdk8', 'jdbc-centos7-openjdk11', 'jdbc-centos7-openjdk17' ] cloud: [ 'AWS' ] category: ['TestCategoryResultSet,TestCategoryOthers,TestCategoryLoader', 'TestCategoryConnection,TestCategoryStatement', 'TestCategoryArrow,TestCategoryCore', 'TestCategoryFips'] steps: @@ -54,6 +55,7 @@ jobs: name: Old JDBC ${{ matrix.category }} on ${{ matrix.image }} runs-on: ubuntu-latest strategy: + fail-fast: false matrix: image: [ 'jdbc-centos6-default' ] cloud: [ 'AWS' ] diff --git a/FIPS/pom.xml b/FIPS/pom.xml index 174bfc791..f98d95ced 100644 --- a/FIPS/pom.xml +++ b/FIPS/pom.xml @@ -973,6 +973,46 @@ + + java-9 + + (9,) + + + + + maven-failsafe-plugin + + + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED + --add-exports=java.base/sun.security.internal.spec=ALL-UNNAMED + --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED + + + + + maven-surefire-plugin + + + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED + --add-exports=java.base/sun.security.internal.spec=ALL-UNNAMED + --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED + + + + + + jenkinsIT diff --git a/ci/_init.sh b/ci/_init.sh index df932feb1..d1e585b85 100755 --- a/ci/_init.sh +++ b/ci/_init.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e export PLATFORM=$(echo $(uname) | tr '[:upper:]' '[:lower:]') export INTERNAL_REPO=nexus.int.snowflakecomputing.com:8086 @@ -15,18 +16,27 @@ mkdir -p $WORKSPACE export DRIVER_NAME=jdbc -# Build images -BUILD_IMAGE_VERSION=1 - # Test Images TEST_IMAGE_VERSION=1 -declare -A BUILD_IMAGE_NAMES=( -) -export BUILD_IMAGE_NAMES - declare -A TEST_IMAGE_NAMES=( - [$DRIVER_NAME-centos6-default]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos6-default-test:$BUILD_IMAGE_VERSION + [$DRIVER_NAME-centos6-default]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos6-default-test:$TEST_IMAGE_VERSION + [$DRIVER_NAME-centos7-openjdk8]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-openjdk8-test:$TEST_IMAGE_VERSION + [$DRIVER_NAME-centos7-openjdk11]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-openjdk11-test:$TEST_IMAGE_VERSION + [$DRIVER_NAME-centos7-openjdk17]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-openjdk17-test:$TEST_IMAGE_VERSION ) export TEST_IMAGE_NAMES +declare -A TEST_IMAGE_DOCKERFILES=( + [$DRIVER_NAME-centos6-default]=jdbc-centos6-default-test + [$DRIVER_NAME-centos7-openjdk8]=jdbc-centos7-openjdk-test + [$DRIVER_NAME-centos7-openjdk11]=jdbc-centos7-openjdk-test + [$DRIVER_NAME-centos7-openjdk17]=jdbc-centos7-openjdk-test +) + +declare -A TEST_IMAGE_BUILD_ARGS=( + [$DRIVER_NAME-centos7-openjdk8]="--target jdbc-centos7-openjdk-yum --build-arg=JDK_PACKAGE=java-1.8.0-openjdk-devel" + [$DRIVER_NAME-centos7-openjdk11]="--target jdbc-centos7-openjdk-yum --build-arg=JDK_PACKAGE=java-11-openjdk-devel" # pragma: allowlist secret + [$DRIVER_NAME-centos7-openjdk17]="--target jdbc-centos7-openjdk17" +) + diff --git a/ci/build.sh b/ci/build.sh index 902523c3d..e933479c5 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -1,4 +1,6 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e + # # Build JDBC driver # diff --git a/ci/image/.gitignore b/ci/image/.gitignore index 523ff310c..3527545e4 100644 --- a/ci/image/.gitignore +++ b/ci/image/.gitignore @@ -1,2 +1,3 @@ pom.xml dependencies/ +*.jar \ No newline at end of file diff --git a/ci/image/Dockerfile.jdbc-centos7-openjdk-test b/ci/image/Dockerfile.jdbc-centos7-openjdk-test new file mode 100644 index 000000000..15e351530 --- /dev/null +++ b/ci/image/Dockerfile.jdbc-centos7-openjdk-test @@ -0,0 +1,91 @@ +FROM centos:7 AS jdbc-centos7-openjdk-base +ARG JDK_PACKAGE + +# update OS +RUN yum -y update && \ + yum -y install epel-release && \ + yum -y install centos-release-scl + +# install Development tools +RUN yum -y groupinstall "Development Tools" && \ + yum -y install zlib-devel which + +# git +RUN curl -o - https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.26.0.tar.gz | tar xfz - && \ + cd git-2.26.0 && \ + ./configure --prefix=/opt/git && make && make install && \ + ln -s /opt/git/bin/git /usr/local/bin/git + +# python +RUN yum -y install rh-python36 +COPY scripts/python3.6.sh /usr/local/bin/python3.6 +COPY scripts/python3.6.sh /usr/local/bin/python3 +RUN chmod a+x /usr/local/bin/python3.6 /usr/local/bin/python3 +COPY scripts/pip.sh /usr/local/bin/pip +RUN chmod a+x /usr/local/bin/pip +RUN pip install -U pip +RUN pip install -U snowflake-connector-python + +# aws +RUN pip install -U awscli +COPY scripts/aws.sh /usr/local/bin/aws +RUN chmod a+x /usr/local/bin/aws + +# zstd +RUN yum -y install zstd + +# jq +RUN yum -y install jq + +# gosu +RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.11/gosu-amd64" +RUN chmod +x /usr/local/bin/gosu +COPY scripts/entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +# Maven +RUN curl -o - https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz | tar xfz - -C /opt && \ + ln -s /opt/apache-maven-3.6.3/bin/mvn /usr/local/bin/mvn + +# workspace +RUN mkdir -p /home/user && \ + chmod 777 /home/user +WORKDIR /home/user + +COPY pom.xml /root +COPY dependencies /root/dependencies + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] + +###### OpenJDK from yum +FROM jdbc-centos7-openjdk-base AS jdbc-centos7-openjdk-yum + +# Java +RUN yum -y install $JDK_PACKAGE + +RUN sed -i /usr/local/bin/entrypoint.sh -e '/^exec/i export JAVA_HOME='$(dirname $(dirname $(readlink $(readlink $(which javac))))) + +RUN cd /root && \ + mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ + -Dnot-self-contained-jar \ + --batch-mode --fail-never compile && \ + mv $HOME/.m2 /home/user && \ + chmod -R 777 /home/user/.m2 + +###### OpenJDK 17 from archive (not available in yum) +FROM jdbc-centos7-openjdk-base AS jdbc-centos7-openjdk17 + +# Java +RUN curl -o - https://download.java.net/java/GA/jdk17.0.2/dfd4a8d0985749f896bed50d7138ee7f/8/GPL/openjdk-17.0.2_linux-x64_bin.tar.gz | tar xfz - -C /opt && \ + ln -s /opt/jdk-17.0.2 /opt/jdk-17 + +RUN sed -i /usr/local/bin/entrypoint.sh -e '/^exec/i export JAVA_HOME=/opt/jdk-17' +RUN sed -i /usr/local/bin/entrypoint.sh -e '/^exec/i export PATH=$JAVA_HOME/bin:$PATH' + +RUN export JAVA_HOME=/opt/jdk-17 && \ + cd /root && \ + mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ + -Dnot-self-contained-jar \ + --batch-mode --fail-never compile && \ + mv $HOME/.m2 /home/user && \ + chmod -R 777 /home/user/.m2 \ No newline at end of file diff --git a/ci/image/build.sh b/ci/image/build.sh index cce0000b6..81c474b4a 100755 --- a/ci/image/build.sh +++ b/ci/image/build.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e # # Build Docker images # @@ -7,22 +8,17 @@ THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $THIS_DIR/../_init.sh cp -p $THIS_DIR/../../pom.xml $THIS_DIR -cp -rp $THIS_DIR/../../dependencies/ $THIS_DIR - -for name in "${!BUILD_IMAGE_NAMES[@]}"; do - docker build \ - --pull \ - --file $THIS_DIR/Dockerfile.$name-build \ - --label snowflake \ - --label $DRIVER_NAME \ - --tag ${BUILD_IMAGE_NAMES[$name]} . -done +mkdir -p dependencies && cp -rp $THIS_DIR/../../dependencies/ $THIS_DIR/dependencies for name in "${!TEST_IMAGE_NAMES[@]}"; do + echo "Building $name" docker build \ + --progress=plain \ + --platform=linux/x86_64 \ --pull \ - --file $THIS_DIR/Dockerfile.$name-test \ + --file $THIS_DIR/Dockerfile.$(echo ${TEST_IMAGE_DOCKERFILES[$name]}) \ --label snowflake \ --label $DRIVER_NAME \ + $(echo ${TEST_IMAGE_BUILD_ARGS[$name]}) \ --tag ${TEST_IMAGE_NAMES[$name]} . done diff --git a/ci/image/update.sh b/ci/image/update.sh index f1642d899..07ffd9fd2 100755 --- a/ci/image/update.sh +++ b/ci/image/update.sh @@ -1,4 +1,6 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e + # # Build Docker images # @@ -12,17 +14,6 @@ source $THIS_DIR/../scripts/login_docker.sh for image in $(docker images --format "{{.ID}},{{.Repository}}:{{.Tag}}" | grep "nexus.int.snowflakecomputing.com" | grep "client-$DRIVER_NAME"); do target_id=$(echo $image | awk -F, '{print $1}') target_name=$(echo $image | awk -F, '{print $2}') - for name in "${!BUILD_IMAGE_NAMES[@]}"; do - if [[ "$target_name" == "${BUILD_IMAGE_NAMES[$name]}" ]]; then - echo $name - docker_hub_image_name=$(echo ${BUILD_IMAGE_NAMES[$name]/$DOCKER_REGISTRY_NAME/snowflakedb}) - set -x - docker tag $target_id $docker_hub_image_name - set +x - docker push "${BUILD_IMAGE_NAMES[$name]}" - docker push "$docker_hub_image_name" - fi - done for name in "${!TEST_IMAGE_NAMES[@]}"; do if [[ "$target_name" == "${TEST_IMAGE_NAMES[$name]}" ]]; then echo $name diff --git a/pom.xml b/pom.xml index 67995a0da..eb8286ded 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ true 5.13.0 1.15.3 - 3.5.6 + 4.11.0 UTF-8 UTF-8 net/snowflake/client/jdbc/internal @@ -502,7 +502,7 @@ org.mockito mockito-core - 3.5.6 + ${mockito.version} test @@ -1396,13 +1396,13 @@ maven-failsafe-plugin - --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED maven-surefire-plugin - --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED @@ -1732,4 +1732,4 @@ - + \ No newline at end of file diff --git a/src/test/java/net/snowflake/client/core/SessionUtilLatestIT.java b/src/test/java/net/snowflake/client/core/SessionUtilLatestIT.java index 2da471f48..14d697371 100644 --- a/src/test/java/net/snowflake/client/core/SessionUtilLatestIT.java +++ b/src/test/java/net/snowflake/client/core/SessionUtilLatestIT.java @@ -171,7 +171,7 @@ public void testForwardedHeaders() throws Throwable { // After login, the only invocation to http should have been with the new // headers. // No calls should have happened without additional headers. - mockedHttpUtil.verify(times(1), httpCalledWithHeaders); + mockedHttpUtil.verify(httpCalledWithHeaders, times(1)); } } @@ -226,7 +226,7 @@ public void testForwardInflightCtx() throws Throwable { // After login, the only invocation to http should have been with the new // headers. // No calls should have happened without additional headers. - mockedHttpUtil.verify(times(1), httpCalledWithHeaders); + mockedHttpUtil.verify(httpCalledWithHeaders, times(1)); } } diff --git a/src/test/java/net/snowflake/client/core/StmtUtilTest.java b/src/test/java/net/snowflake/client/core/StmtUtilTest.java index e59920206..5b7e2703b 100644 --- a/src/test/java/net/snowflake/client/core/StmtUtilTest.java +++ b/src/test/java/net/snowflake/client/core/StmtUtilTest.java @@ -88,7 +88,7 @@ public void testForwardedHeaders() throws Throwable { // After login, the only invocation to http should have been with the new // headers. // No calls should have happened without additional headers. - mockedHttpUtil.verify(times(1), httpCalledWithHeaders); + mockedHttpUtil.verify(httpCalledWithHeaders, times(1)); } } diff --git a/src/test/java/net/snowflake/client/jdbc/ConnectionWithOCSPModeIT.java b/src/test/java/net/snowflake/client/jdbc/ConnectionWithOCSPModeIT.java index 29561d16a..03a58517f 100644 --- a/src/test/java/net/snowflake/client/jdbc/ConnectionWithOCSPModeIT.java +++ b/src/test/java/net/snowflake/client/jdbc/ConnectionWithOCSPModeIT.java @@ -23,6 +23,7 @@ import net.snowflake.client.core.SFTrustManager; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -392,6 +393,8 @@ public void testOCSPResponder403FailClosed() { /** Test Certificate Expired. Will fail in both FAIL_OPEN and FAIL_CLOSED. */ @Test + @Ignore("Issuer of root CA expired") + // https://support.sectigo.com/articles/Knowledge/Sectigo-AddTrust-External-CA-Root-Expiring-May-30-2020 public void testExpiredCert() { try { DriverManager.getConnection(