Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

no-snow Parallelize e2e jar tests #767

Merged
merged 1 commit into from
Jun 11, 2024
Merged

no-snow Parallelize e2e jar tests #767

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .github/workflows/End2EndTest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,14 @@ jobs:
continue-on-error: false
run: mvn -DghActionsIT verify --batch-mode
build-e2e-jar-test:
name: E2E JAR Test - Cloud ${{ matrix.snowflake_cloud }}
name: e2e-jar-test cloud=${{ matrix.snowflake_cloud }} test_type=${{ matrix.test_type }} java=${{ matrix.java_path_env_var }}
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
snowflake_cloud: [ 'AWS', 'AZURE', 'GCP' ]
test_type: [ 'shaded', 'unshaded', 'fips']
java_path_env_var: [ "JAVA_HOME_8_X64", "JAVA_HOME_11_X64", "JAVA_HOME_17_X64", "JAVA_HOME_21_X64" ]
steps:
- name: Checkout Code
uses: actions/checkout@v2
Expand All @@ -92,4 +94,7 @@ jobs:
DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }}
run: ./scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }}
- name: Run E2E JAR Test
env:
test_type: ${{ matrix.test_type }}
java_path_env_var: ${{ matrix.java_path_env_var }}
run: ./e2e-jar-test/run_e2e_jar_test.sh
2 changes: 1 addition & 1 deletion e2e-jar-test/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<dependency>
<groupId>net.snowflake</groupId>
<artifactId>snowflake-ingest-sdk</artifactId>
<version>2.1.1</version>
<version>2.1.2-SNAPSHOT</version>
</dependency>

<dependency>
Expand Down
124 changes: 52 additions & 72 deletions e2e-jar-test/run_e2e_jar_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,101 +2,81 @@

set -euo pipefail

maven_repo_dir=$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
sdk_repo_dir="${maven_repo_dir}/net/snowflake/snowflake-ingest-sdk"

## This script tests the SDK JARs end-to-end, i.e. not using integration tests from within the project, but from an
## external Maven project, which depends on the SDK deployed into the local maven repository. The following SDK variants are tested:
## 1. Shaded jar
## 2. Unshaded jar
## 3. FIPS-compliant jar, i.e. unshaded jar without snowflake-jdbc and bouncy castle dependencies, but with snowflake-jdbc-fips depedency

maven_repo_dir=$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
sdk_repo_dir="${maven_repo_dir}/net/snowflake/snowflake-ingest-sdk"

cp profile.json e2e-jar-test/standard
cp profile.json e2e-jar-test/fips

echo "###################"
echo "# TEST SHADED JAR #"
echo "###################"

# Remove the SDK from local maven repository
rm -fr $sdk_repo_dir

# Prepare pom.xml for shaded JAR
project_version=$(./scripts/get_project_info_from_pom.py pom.xml version)
./scripts/update_project_version.py public_pom.xml $project_version > generated_public_pom.xml

# Build shaded SDK always with Java 8
echo "Building shaded SDK"
mvn clean package -DskipTests=true --batch-mode --show-version

# Install shaded SDK JARs into local maven repository
mvn install:install-file -Dfile=target/snowflake-ingest-sdk.jar -DpomFile=generated_public_pom.xml

# Run e2e tests with all supported LTS Java versions
cd e2e-jar-test

echo "Testing shaded JAR with Java 8"
mvn --show-version clean verify -pl standard -am # Test with Java 8

echo "Testing shaded JAR with Java 11"
JAVA_HOME="${JAVA_HOME_11_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 11
test_type="${test_type:-}"
java_path_for_test_execution="${!java_path_env_var}"

echo "Testing shaded JAR with Java 17"
JAVA_HOME="${JAVA_HOME_17_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 17
# Cleanup ingest SDK in the local maven repository
rm -fr "${sdk_repo_dir}"

echo "Testing shaded JAR with Java 21"
JAVA_HOME="${JAVA_HOME_21_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 21
cd ..
if [[ "${test_type}" == "shaded" ]]; then
echo "###################"
echo "# TEST SHADED JAR #"
echo "###################"

echo "#####################"
echo "# TEST UNSHADED JAR #"
echo "#####################"
# Prepare pom.xml for shaded JAR
project_version=$(./scripts/get_project_info_from_pom.py pom.xml version)
./scripts/update_project_version.py public_pom.xml $project_version > generated_public_pom.xml

# Remove the SDK from local maven repository
rm -r $sdk_repo_dir
# Build shaded SDK always with Java 8
echo "Building shaded SDK"
JAVA_HOME="${JAVA_HOME_8_X64}" mvn clean package -DskipTests=true --batch-mode --show-version

# Install unshaded SDK into local maven repository with Java 8
echo "Building unshaded SDK"
mvn clean install -Dnot-shadeDep -DskipTests=true --batch-mode --show-version
# Install shaded SDK JARs into local maven repository
JAVA_HOME="${JAVA_HOME_8_X64}" mvn install:install-file -Dfile=target/snowflake-ingest-sdk.jar -DpomFile=generated_public_pom.xml

# Run e2e tests with all supported LTS Java versions
cd e2e-jar-test
# Run e2e tests with all supported LTS Java versions
cd e2e-jar-test

echo "Testing unshaded JAR with Java 8"
mvn --show-version clean verify -pl standard -am # Test with Java 8
echo "Testing shaded JAR with ${java_path_for_test_execution}"
JAVA_HOME="${java_path_for_test_execution}" mvn --show-version clean verify -pl standard -am
cd ..
elif [[ "${test_type}" == "unshaded" ]]; then
echo "#####################"
echo "# TEST UNSHADED JAR #"
echo "#####################"

echo "Testing unshaded JAR with Java 11"
JAVA_HOME="${JAVA_HOME_11_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 11
# Install unshaded SDK into local maven repository with Java 8
echo "Building unshaded SDK"
JAVA_HOME="${JAVA_HOME_8_X64}" mvn clean install -Dnot-shadeDep -DskipTests=true --batch-mode --show-version

echo "Testing unshaded JAR with Java 17"
JAVA_HOME="${JAVA_HOME_17_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 17
# Run e2e tests with all supported LTS Java versions
cd e2e-jar-test

echo "Testing unshaded JAR with Java 21"
JAVA_HOME="${JAVA_HOME_21_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 21
cd ..
echo "Testing unshaded JAR with ${java_path_for_test_execution}"
JAVA_HOME="${java_path_for_test_execution}" mvn --show-version clean verify -pl standard -am
cd ..
elif [[ "${test_type}" == "fips" ]]; then
echo "##############"
echo "# TEST FIPS #"
echo "##############"
# Install unshaded SDK into local maven repository with Java
echo "Building unshaded SDK for FIPS"
JAVA_HOME="${JAVA_HOME_8_X64}" mvn clean install -Dnot-shadeDep -DskipTests=true --batch-mode --show-version

echo "##############"
echo "# TEST FIPS #"
echo "##############"
# Remove the SDK from local maven repository
rm -r $sdk_repo_dir
# Run e2e tests on the FIPS module with all supported LTS Java versions
cd e2e-jar-test

# Install unshaded SDK into local maven repository with Java
echo "Building unshaded SDK for FIPS"
mvn clean install -Dnot-shadeDep -DskipTests=true --batch-mode --show-version
echo "Testing FIPS JAR with ${java_path_for_test_execution}"
JAVA_HOME="${java_path_for_test_execution}" mvn --show-version clean verify -pl standard -am
cd ..
else
echo "Unsupported value of \$test_type variable: \"${test_type}\"" >&2
exit 1
fi

# Run e2e tests on the FIPS module with all supported LTS Java versions
cd e2e-jar-test

echo "Testing FIPS JAR with Java 8"
mvn --show-version clean verify -pl standard -am # Test with Java 8

echo "Testing FIPS JAR with Java 11"
JAVA_HOME="${JAVA_HOME_11_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 11

echo "Testing FIPS JAR with Java 17"
JAVA_HOME="${JAVA_HOME_17_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 17

echo "Testing FIPS JAR with Java 21"
JAVA_HOME="${JAVA_HOME_21_X64}" mvn --show-version clean verify -pl standard -am # Test with Java 21
cd ..
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<!-- Arifact name and version information -->
<groupId>net.snowflake</groupId>
<artifactId>snowflake-ingest-sdk</artifactId>
<version>2.1.1</version>
<version>2.1.2-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Snowflake Ingest SDK</name>
<description>Snowflake Ingest SDK</description>
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public class RequestBuilder {
// Don't change!
public static final String CLIENT_NAME = "SnowpipeJavaSDK";

public static final String DEFAULT_VERSION = "2.1.1";
public static final String DEFAULT_VERSION = "2.1.2-SNAPSHOT";

public static final String JAVA_USER_AGENT = "JAVA";

Expand Down
Loading