SNOW-1819816 Add an IT for large parquet file #2660
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Snowpipe Java SDK | |
| on: | |
| push: | |
| branches: [ master ] | |
| pull_request: | |
| branches: '**' | |
| jobs: | |
| build: | |
| name: Build & Test - JDK ${{ matrix.java }}, Cloud ${{ matrix.snowflake_cloud }} | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast | |
| matrix: | |
| java: [ 8 ] | |
| snowflake_cloud: [ 'AWS', 'AZURE', 'GCP' ] | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v2 | |
| - name: Install Java ${{ matrix.java }} | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ matrix.java }} | |
| cache: maven | |
| - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }} | |
| env: | |
| DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }} | |
| run: | | |
| ./scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }} | |
| - name: Unit & Integration Test against Cloud ${{ matrix.snowflake_cloud }} | |
| env: | |
| JACOCO_COVERAGE: true | |
| WHITESOURCE_API_KEY: ${{ secrets.WHITESOURCE_API_KEY }} | |
| continue-on-error: false | |
| run: | | |
| ./scripts/run_gh_actions.sh -Dfailsafe.excludedGroups="net.snowflake.ingest.IcebergIT" | |
| - name: Code Coverage | |
| uses: codecov/codecov-action@v1 | |
| build-windows: | |
| name: Build & Test - Windows, JDK ${{ matrix.java }}, Cloud ${{ matrix.snowflake_cloud }} | |
| runs-on: windows-2022 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [ 8 ] | |
| snowflake_cloud: [ 'AWS', 'AZURE', 'GCP' ] | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v2 | |
| - name: Install Java ${{ matrix.java }} | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ matrix.java }} | |
| cache: maven | |
| - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }} on Windows Powershell | |
| env: | |
| DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }} | |
| shell: pwsh | |
| run: | | |
| ./scripts/decrypt_secret_windows.ps1 -SnowflakeDeployment '${{ matrix.snowflake_cloud }}' | |
| - name: Unit & Integration Test (Windows) | |
| continue-on-error: false | |
| run: | | |
| mvn -DghActionsIT -D"failsafe.excludedGroups"="net.snowflake.ingest.IcebergIT" verify --batch-mode | |
| build-iceberg: | |
| name: Build & Test Streaming Iceberg - JDK ${{ matrix.java }}, Cloud ${{ matrix.snowflake_cloud }} | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast | |
| matrix: | |
| java: [ 8 ] | |
| snowflake_cloud: [ 'AWS', 'AZURE' ] | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v2 | |
| - name: Install Java ${{ matrix.java }} | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ matrix.java }} | |
| cache: maven | |
| - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }} | |
| env: | |
| DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }} | |
| run: | | |
| ./scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }} | |
| - name: Unit & Integration Test against Cloud ${{ matrix.snowflake_cloud }} | |
| env: | |
| JACOCO_COVERAGE: true | |
| WHITESOURCE_API_KEY: ${{ secrets.WHITESOURCE_API_KEY }} | |
| continue-on-error: false | |
| run: | | |
| ./scripts/run_gh_actions.sh -Dfailsafe.groups="net.snowflake.ingest.IcebergIT" | |
| - name: Code Coverage | |
| uses: codecov/codecov-action@v1 | |
| build-iceberg-windows: | |
| name: Build & Test - Streaming Iceberg Windows, JDK ${{ matrix.java }}, Cloud ${{ matrix.snowflake_cloud }} | |
| runs-on: windows-2022 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [ 8 ] | |
| snowflake_cloud: [ 'AWS', 'AZURE' ] | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v2 | |
| - name: Install Java ${{ matrix.java }} | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ matrix.java }} | |
| cache: maven | |
| - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }} on Windows Powershell | |
| env: | |
| DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }} | |
| shell: pwsh | |
| run: | | |
| ./scripts/decrypt_secret_windows.ps1 -SnowflakeDeployment '${{ matrix.snowflake_cloud }}' | |
| - name: Unit & Integration Test (Windows) | |
| continue-on-error: false | |
| run: | | |
| mvn -DghActionsIT -"Dfailsafe.groups"="net.snowflake.ingest.IcebergIT" verify --batch-mode | |
| build-e2e-jar-test: | |
| name: e2e-jar-test cloud=${{ matrix.snowflake_cloud }} test_type=${{ matrix.test_type }} java=${{ matrix.java_path_env_var }} | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| snowflake_cloud: [ 'AWS', 'AZURE', 'GCP' ] | |
| test_type: [ 'shaded', 'unshaded', 'fips'] | |
| java_path_env_var: [ "JAVA_HOME_8_X64", "JAVA_HOME_11_X64", "JAVA_HOME_17_X64", "JAVA_HOME_21_X64" ] | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v2 | |
| - name: Install Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: | # Install all LTS java versions, the last mentioned one here will be the default | |
| 21 | |
| 17 | |
| 11 | |
| 8 | |
| cache: maven | |
| - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }} | |
| env: | |
| DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }} | |
| run: ./scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }} | |
| - name: Run E2E JAR Test | |
| env: | |
| test_type: ${{ matrix.test_type }} | |
| java_path_env_var: ${{ matrix.java_path_env_var }} | |
| run: ./e2e-jar-test/run_e2e_jar_test.sh |