From b0146dd5879d3bce9eb921b888441260f693b710 Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Fri, 8 Nov 2024 17:56:47 -0800 Subject: [PATCH] Revert "Introduce Iceberg streaming merge gate" (#897) Revert "Introduce Iceberg streaming merge gate (#896)" This reverts commit cd0be31481e39f1186bd5925c23bb1be59cec26f. --- .github/workflows/End2EndTest.yml | 66 +------------------ pom.xml | 4 -- scripts/run_gh_actions.sh | 5 -- .../java/net/snowflake/ingest/IcebergIT.java | 8 --- .../internal/datatypes/IcebergDateTimeIT.java | 5 +- .../datatypes/IcebergLogicalTypesIT.java | 9 +-- .../datatypes/IcebergNumericTypesIT.java | 5 +- .../internal/datatypes/IcebergStringIT.java | 9 +-- .../datatypes/IcebergStructuredIT.java | 5 +- .../internal/it/IcebergColumnNamesIT.java | 5 +- 10 files changed, 14 insertions(+), 107 deletions(-) delete mode 100644 src/test/java/net/snowflake/ingest/IcebergIT.java diff --git a/.github/workflows/End2EndTest.yml b/.github/workflows/End2EndTest.yml index 78c0c28ca..0a47d7bd1 100644 --- a/.github/workflows/End2EndTest.yml +++ b/.github/workflows/End2EndTest.yml @@ -37,7 +37,7 @@ jobs: WHITESOURCE_API_KEY: ${{ secrets.WHITESOURCE_API_KEY }} continue-on-error: false run: | - ./scripts/run_gh_actions.sh -Dfailsafe.excludedGroups="net.snowflake.ingest.IcebergIT" + ./scripts/run_gh_actions.sh - name: Code Coverage uses: codecov/codecov-action@v1 @@ -66,69 +66,7 @@ jobs: ./scripts/decrypt_secret_windows.ps1 -SnowflakeDeployment '${{ matrix.snowflake_cloud }}' - name: Unit & Integration Test (Windows) continue-on-error: false - run: | - mvn -DghActionsIT -D"failsafe.excludedGroups"="net.snowflake.ingest.IcebergIT" verify --batch-mode - build-iceberg: - name: Build & Test Streaming Iceberg - JDK ${{ matrix.java }}, Cloud ${{ matrix.snowflake_cloud }} - runs-on: ubuntu-20.04 - strategy: - fail-fast: false # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast - matrix: - java: [ 8 ] - snowflake_cloud: [ 'AWS' ] - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Install Java ${{ matrix.java }} - uses: actions/setup-java@v2 - with: - distribution: temurin - java-version: ${{ matrix.java }} - cache: maven - - - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }} - env: - DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }} - run: | - ./scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }} - - - name: Unit & Integration Test against Cloud ${{ matrix.snowflake_cloud }} - env: - JACOCO_COVERAGE: true - WHITESOURCE_API_KEY: ${{ secrets.WHITESOURCE_API_KEY }} - continue-on-error: false - run: | - ./scripts/run_gh_actions.sh -Dfailsafe.groups="net.snowflake.ingest.IcebergIT" - - - name: Code Coverage - uses: codecov/codecov-action@v1 - build-iceberg-windows: - name: Build & Test - Streaming Iceberg Windows, JDK ${{ matrix.java }}, Cloud ${{ matrix.snowflake_cloud }} - runs-on: windows-2022 - strategy: - fail-fast: false - matrix: - java: [ 8 ] - snowflake_cloud: [ 'AWS' ] - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Install Java ${{ matrix.java }} - uses: actions/setup-java@v2 - with: - distribution: temurin - java-version: ${{ matrix.java }} - cache: maven - - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }} on Windows Powershell - env: - DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }} - shell: pwsh - run: | - ./scripts/decrypt_secret_windows.ps1 -SnowflakeDeployment '${{ matrix.snowflake_cloud }}' - - name: Unit & Integration Test (Windows) - continue-on-error: false - run: | - mvn -DghActionsIT -"Dfailsafe.groups"="net.snowflake.ingest.IcebergIT" verify --batch-mode + run: mvn -DghActionsIT verify --batch-mode build-e2e-jar-test: name: e2e-jar-test cloud=${{ matrix.snowflake_cloud }} test_type=${{ matrix.test_type }} java=${{ matrix.java_path_env_var }} runs-on: ubuntu-20.04 diff --git a/pom.xml b/pom.xml index cddaea8a2..726afe8d2 100644 --- a/pom.xml +++ b/pom.xml @@ -1406,10 +1406,6 @@ org.apache.maven.plugins maven-failsafe-plugin - - ${failsafe.groups} - ${failsafe.excludedGroups} - diff --git a/scripts/run_gh_actions.sh b/scripts/run_gh_actions.sh index c6f9f99ed..49895010b 100755 --- a/scripts/run_gh_actions.sh +++ b/scripts/run_gh_actions.sh @@ -1,9 +1,5 @@ #!/bin/bash -e -# -# Copyright (c) 2024 Snowflake Computing Inc. All rights reserved. -# - set -o pipefail # Build and install shaded JAR first. check_content.sh runs here. @@ -13,7 +9,6 @@ PARAMS=() PARAMS+=("-DghActionsIT") # testing will not need shade dep. otherwise codecov cannot work PARAMS+=("-Dnot-shadeDep") -PARAMS+=($1) [[ -n "$JACOCO_COVERAGE" ]] && PARAMS+=("-Djacoco.skip.instrument=false") # verify phase is after test/integration-test phase, which means both unit test # and integration test will be run diff --git a/src/test/java/net/snowflake/ingest/IcebergIT.java b/src/test/java/net/snowflake/ingest/IcebergIT.java deleted file mode 100644 index 1f6ffe101..000000000 --- a/src/test/java/net/snowflake/ingest/IcebergIT.java +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved. - */ - -package net.snowflake.ingest; - -/** Interface for Iceberg Integration Tests groups */ -public interface IcebergIT {} diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergDateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergDateTimeIT.java index db17d68ec..eeb0cee4b 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergDateTimeIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergDateTimeIT.java @@ -14,18 +14,17 @@ import java.time.OffsetTime; import java.time.ZoneOffset; import java.util.Arrays; -import net.snowflake.ingest.IcebergIT; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; import org.assertj.core.api.Assertions; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; -import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -@Category(IcebergIT.class) +@Ignore("This test can be enabled after server side Iceberg EP support is released") @RunWith(Parameterized.class) public class IcebergDateTimeIT extends AbstractDataTypeTest { @Parameterized.Parameters(name = "compressionAlgorithm={0}, icebergSerializationPolicy={1}") diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergLogicalTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergLogicalTypesIT.java index 63421d68c..e6d54418e 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergLogicalTypesIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergLogicalTypesIT.java @@ -1,22 +1,17 @@ -/* - * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved. - */ - package net.snowflake.ingest.streaming.internal.datatypes; import java.util.Arrays; -import net.snowflake.ingest.IcebergIT; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; import org.assertj.core.api.Assertions; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; -import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -@Category(IcebergIT.class) +@Ignore("This test can be enabled after server side Iceberg EP support is released") @RunWith(Parameterized.class) public class IcebergLogicalTypesIT extends AbstractDataTypeTest { @Parameterized.Parameters(name = "compressionAlgorithm={0}, icebergSerializationPolicy={1}") diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergNumericTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergNumericTypesIT.java index 21e28fc87..d5d436f6f 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergNumericTypesIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergNumericTypesIT.java @@ -9,21 +9,20 @@ import java.util.Arrays; import java.util.List; import java.util.Random; -import net.snowflake.ingest.IcebergIT; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; import org.apache.commons.text.RandomStringGenerator; import org.assertj.core.api.Assertions; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; -import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Category(IcebergIT.class) +@Ignore("This test can be enabled after server side Iceberg EP support is released") @RunWith(Parameterized.class) public class IcebergNumericTypesIT extends AbstractDataTypeTest { @Parameterized.Parameters(name = "compressionAlgorithm={0}, icebergSerializationPolicy={1}") diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergStringIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergStringIT.java index 7b25872d8..2e0989994 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergStringIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergStringIT.java @@ -1,24 +1,19 @@ -/* - * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved. - */ - package net.snowflake.ingest.streaming.internal.datatypes; import java.math.BigDecimal; import java.util.Arrays; -import net.snowflake.ingest.IcebergIT; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; import org.apache.commons.lang3.StringUtils; import org.assertj.core.api.Assertions; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; -import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -@Category(IcebergIT.class) +@Ignore("This test can be enabled after server side Iceberg EP support is released") @RunWith(Parameterized.class) public class IcebergStringIT extends AbstractDataTypeTest { @Parameterized.Parameters(name = "compressionAlgorithm={0}, icebergSerializationPolicy={1}") diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergStructuredIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergStructuredIT.java index 2995d1dc9..8d3071518 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergStructuredIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergStructuredIT.java @@ -15,7 +15,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import net.snowflake.ingest.IcebergIT; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.streaming.InsertValidationResponse; import net.snowflake.ingest.streaming.OpenChannelRequest; @@ -25,12 +24,12 @@ import net.snowflake.ingest.utils.SFException; import org.assertj.core.api.Assertions; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; -import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -@Category(IcebergIT.class) +@Ignore("This test can be enabled after server side Iceberg EP support is released") @RunWith(Parameterized.class) public class IcebergStructuredIT extends AbstractDataTypeTest { @Parameterized.Parameters(name = "compressionAlgorithm={0}, icebergSerializationPolicy={1}") diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/it/IcebergColumnNamesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/it/IcebergColumnNamesIT.java index 58e368fc1..8051a9467 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/it/IcebergColumnNamesIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/it/IcebergColumnNamesIT.java @@ -4,12 +4,11 @@ package net.snowflake.ingest.streaming.internal.it; -import net.snowflake.ingest.IcebergIT; import net.snowflake.ingest.utils.Constants.IcebergSerializationPolicy; import org.junit.Before; -import org.junit.experimental.categories.Category; +import org.junit.Ignore; -@Category(IcebergIT.class) +@Ignore("Enable this after the Iceberg testing on GCS / Azure is ready") public class IcebergColumnNamesIT extends ColumnNamesITBase { @Before public void before() throws Exception {