Revert "Introduce Iceberg streaming merge gate" (#897)

Revert "Introduce Iceberg streaming merge gate (#896)" This reverts commit cd0be31.
snowflakedb · Nov 9, 2024 · b0146dd · b0146dd
1 parent cd0be31
commit b0146dd
Show file tree

Hide file tree

Showing 10 changed files with 14 additions and 107 deletions.
diff --git a/.github/workflows/End2EndTest.yml b/.github/workflows/End2EndTest.yml
@@ -37,7 +37,7 @@ jobs:
           WHITESOURCE_API_KEY: ${{ secrets.WHITESOURCE_API_KEY }}
         continue-on-error: false
         run: |
-          ./scripts/run_gh_actions.sh -Dfailsafe.excludedGroups="net.snowflake.ingest.IcebergIT"
+          ./scripts/run_gh_actions.sh
 
       - name: Code Coverage
         uses: codecov/codecov-action@v1
@@ -66,69 +66,7 @@ jobs:
           ./scripts/decrypt_secret_windows.ps1 -SnowflakeDeployment '${{ matrix.snowflake_cloud }}'
       - name: Unit & Integration Test (Windows)
         continue-on-error: false
-        run: |
-          mvn -DghActionsIT -D"failsafe.excludedGroups"="net.snowflake.ingest.IcebergIT" verify --batch-mode
-  build-iceberg:
-    name: Build & Test Streaming Iceberg - JDK ${{ matrix.java }}, Cloud ${{ matrix.snowflake_cloud }}
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast
-      matrix:
-        java: [ 8 ]
-        snowflake_cloud: [ 'AWS' ]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v2
-      - name: Install Java ${{ matrix.java }}
-        uses: actions/setup-java@v2
-        with:
-          distribution: temurin
-          java-version: ${{ matrix.java }}
-          cache: maven
-
-      - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }}
-        env:
-          DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }}
-        run: |
-          ./scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }}
-
-      - name: Unit & Integration Test against Cloud ${{ matrix.snowflake_cloud }}
-        env:
-          JACOCO_COVERAGE: true
-          WHITESOURCE_API_KEY: ${{ secrets.WHITESOURCE_API_KEY }}
-        continue-on-error: false
-        run: |
-          ./scripts/run_gh_actions.sh -Dfailsafe.groups="net.snowflake.ingest.IcebergIT"
-
-      - name: Code Coverage
-        uses: codecov/codecov-action@v1
-  build-iceberg-windows:
-    name: Build & Test - Streaming Iceberg Windows, JDK ${{ matrix.java }}, Cloud ${{ matrix.snowflake_cloud }}
-    runs-on: windows-2022
-    strategy:
-      fail-fast: false
-      matrix:
-        java: [ 8 ]
-        snowflake_cloud: [ 'AWS' ]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v2
-      - name: Install Java ${{ matrix.java }}
-        uses: actions/setup-java@v2
-        with:
-          distribution: temurin
-          java-version: ${{ matrix.java }}
-          cache: maven
-      - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }} on Windows Powershell
-        env:
-          DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }}
-        shell: pwsh
-        run: |
-          ./scripts/decrypt_secret_windows.ps1 -SnowflakeDeployment '${{ matrix.snowflake_cloud }}'
-      - name: Unit & Integration Test (Windows)
-        continue-on-error: false
-        run: |
-          mvn -DghActionsIT -"Dfailsafe.groups"="net.snowflake.ingest.IcebergIT" verify --batch-mode
+        run: mvn -DghActionsIT verify --batch-mode
   build-e2e-jar-test:
     name: e2e-jar-test cloud=${{ matrix.snowflake_cloud }} test_type=${{ matrix.test_type }} java=${{ matrix.java_path_env_var }}
     runs-on: ubuntu-20.04

diff --git a/pom.xml b/pom.xml
@@ -1406,10 +1406,6 @@
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-failsafe-plugin</artifactId>
-            <configuration>
-              <groups>${failsafe.groups}</groups>
-              <excludedGroups>${failsafe.excludedGroups}</excludedGroups>
-            </configuration>
             <executions>
               <execution>
                 <goals>

diff --git a/scripts/run_gh_actions.sh b/scripts/run_gh_actions.sh
@@ -1,9 +1,5 @@
 #!/bin/bash -e
 
-#
-# Copyright (c) 2024 Snowflake Computing Inc. All rights reserved.
-#
-
 set -o pipefail
 
 # Build and install shaded JAR first. check_content.sh runs here.
@@ -13,7 +9,6 @@ PARAMS=()
 PARAMS+=("-DghActionsIT")
 # testing will not need shade dep. otherwise codecov cannot work
 PARAMS+=("-Dnot-shadeDep")
-PARAMS+=($1)
 [[ -n "$JACOCO_COVERAGE" ]] && PARAMS+=("-Djacoco.skip.instrument=false")
 # verify phase is after test/integration-test phase, which means both unit test
 # and integration test will be run

diff --git a/src/test/java/net/snowflake/ingest/IcebergIT.java b/src/test/java/net/snowflake/ingest/IcebergIT.java
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergDateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergDateTimeIT.java
@@ -14,18 +14,17 @@
 import java.time.OffsetTime;
 import java.time.ZoneOffset;
 import java.util.Arrays;
-import net.snowflake.ingest.IcebergIT;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import org.assertj.core.api.Assertions;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
-import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
-@Category(IcebergIT.class)
+@Ignore("This test can be enabled after server side Iceberg EP support is released")
 @RunWith(Parameterized.class)
 public class IcebergDateTimeIT extends AbstractDataTypeTest {
   @Parameterized.Parameters(name = "compressionAlgorithm={0}, icebergSerializationPolicy={1}")

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergLogicalTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergLogicalTypesIT.java
@@ -1,22 +1,17 @@
-/*
- * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved.
- */
-
 package net.snowflake.ingest.streaming.internal.datatypes;
 
 import java.util.Arrays;
-import net.snowflake.ingest.IcebergIT;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import org.assertj.core.api.Assertions;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
-import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
-@Category(IcebergIT.class)
+@Ignore("This test can be enabled after server side Iceberg EP support is released")
 @RunWith(Parameterized.class)
 public class IcebergLogicalTypesIT extends AbstractDataTypeTest {
   @Parameterized.Parameters(name = "compressionAlgorithm={0}, icebergSerializationPolicy={1}")

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergNumericTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergNumericTypesIT.java
@@ -9,21 +9,20 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
-import net.snowflake.ingest.IcebergIT;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import org.apache.commons.text.RandomStringGenerator;
 import org.assertj.core.api.Assertions;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
-import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-@Category(IcebergIT.class)
+@Ignore("This test can be enabled after server side Iceberg EP support is released")
 @RunWith(Parameterized.class)
 public class IcebergNumericTypesIT extends AbstractDataTypeTest {
   @Parameterized.Parameters(name = "compressionAlgorithm={0}, icebergSerializationPolicy={1}")

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergStringIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergStringIT.java
@@ -1,24 +1,19 @@
-/*
- * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved.
- */
-
 package net.snowflake.ingest.streaming.internal.datatypes;
 
 import java.math.BigDecimal;
 import java.util.Arrays;
-import net.snowflake.ingest.IcebergIT;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import org.apache.commons.lang3.StringUtils;
 import org.assertj.core.api.Assertions;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
-import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
-@Category(IcebergIT.class)
+@Ignore("This test can be enabled after server side Iceberg EP support is released")
 @RunWith(Parameterized.class)
 public class IcebergStringIT extends AbstractDataTypeTest {
   @Parameterized.Parameters(name = "compressionAlgorithm={0}, icebergSerializationPolicy={1}")

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergStructuredIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/IcebergStructuredIT.java
@@ -15,7 +15,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
-import net.snowflake.ingest.IcebergIT;
 import net.snowflake.ingest.TestUtils;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
@@ -25,12 +24,12 @@
 import net.snowflake.ingest.utils.SFException;
 import org.assertj.core.api.Assertions;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
-import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
-@Category(IcebergIT.class)
+@Ignore("This test can be enabled after server side Iceberg EP support is released")
 @RunWith(Parameterized.class)
 public class IcebergStructuredIT extends AbstractDataTypeTest {
   @Parameterized.Parameters(name = "compressionAlgorithm={0}, icebergSerializationPolicy={1}")

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/it/IcebergColumnNamesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/it/IcebergColumnNamesIT.java
@@ -4,12 +4,11 @@
 
 package net.snowflake.ingest.streaming.internal.it;
 
-import net.snowflake.ingest.IcebergIT;
 import net.snowflake.ingest.utils.Constants.IcebergSerializationPolicy;
 import org.junit.Before;
-import org.junit.experimental.categories.Category;
+import org.junit.Ignore;
 
-@Category(IcebergIT.class)
+@Ignore("Enable this after the Iceberg testing on GCS / Azure is ready")
 public class IcebergColumnNamesIT extends ColumnNamesITBase {
   @Before
   public void before() throws Exception {