From a438a4370fb5173c02e130571c33238798ab0303 Mon Sep 17 00:00:00 2001 From: Berzan Mikaili Date: Thu, 5 Sep 2024 13:11:32 +0000 Subject: [PATCH] Fix tests --- dependency-reduced-pom.xml | 1175 ----------------- ...nowflakeStreamingIngestClientInternal.java | 7 +- .../SnowflakeStreamingIngestClientTest.java | 9 + 3 files changed, 15 insertions(+), 1176 deletions(-) delete mode 100644 dependency-reduced-pom.xml diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml deleted file mode 100644 index 05f2b442b..000000000 --- a/dependency-reduced-pom.xml +++ /dev/null @@ -1,1175 +0,0 @@ - - - 4.0.0 - net.snowflake - snowflake-ingest-sdk - Snowflake Ingest SDK - 2.2.0 - Snowflake Ingest SDK - https://www.snowflake.net/ - - - Snowflake Support Team - snowflake-java@snowflake.net - Snowflake Computing - https://www.snowflake.net - - - - - The Apache Software License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - - - - scm:git:git://github.com/snowflakedb/snowflake-ingest-java - https://github.com/snowflakedb/snowflake-ingest-java/tree/master - - - - - true - src/main/resources - - - ${license.processing.resourcesRoot} - - - ${project.artifactId} - - - - maven-dependency-plugin - 3.6.0 - - - maven-deploy-plugin - - true - - - - maven-enforcer-plugin - 3.4.1 - - - com.google.cloud.tools - linkage-checker-enforcer-rules - 1.5.13 - - - org.codehaus.mojo - extra-enforcer-rules - 1.3 - - - aether-util - org.eclipse.aether - - - - - - - maven-failsafe-plugin - 3.0.0-M5 - - - maven-surefire-plugin - 3.2.5 - - - org.codehaus.mojo - exec-maven-plugin - 3.2.0 - - - org.jacoco - jacoco-maven-plugin - ${jacoco.version} - - - pre-unit-test - - prepare-agent - - - target/jacoco-ut.exec - - - - post-unit-test - test - - report - - - target/jacoco-ut.exec - target/jacoco-ut - - - - - ${jacoco.skip.instrument} - - **/*SnowflakeStreamingIngestExample* - **/*SnowflakeIngestBasicExample* - **/*IngestExampleHelper* - - - - - - - - com.github.ekryd.sortpom - sortpom-maven-plugin - 3.0.1 - - - validate - - verify - - - - - false - false - true - scope,groupId,artifactId - groupId,artifactId - true - true - groupId,artifactId - true - stop - strict - - - - maven-assembly-plugin - - - - true - net.snowflake.ingest.SimpleIngestManager - - - - jar-with-dependencies - - - - - maven-compiler-plugin - 2.5.1 - true - - 1.8 - 1.8 - - - - maven-dependency-plugin - - - analyze - - analyze-only - - - true - true - - org.apache.commons:commons-compress - org.apache.commons:commons-configuration2 - - - - - - - maven-enforcer-plugin - - - enforce-best-practices - - enforce - - - - - - META-INF/versions/*/org/bouncycastle/* - META-INF/versions/*/com/fasterxml/jackson/core/io/doubleparser/* - - true - true - - - - - - - - - - enforce-maven - - enforce - - - - - 3.5 - - - - - - - - maven-javadoc-plugin - 2.10.4 - - - attach-javadocs - - jar - - - -Xdoclint:none - 8 - - - - - - maven-source-plugin - 3.0.1 - - - attach-sources - - jar - - - - - - maven-surefire-plugin - - - **/TestSimpleIngestLocal.java - - - - - org.codehaus.mojo - license-maven-plugin - 2.0.1 - - - add-third-party - package - - add-third-party - - - - - failFast - - Apache License 2.0 - BSD 2-Clause License - 3-Clause BSD License - The MIT License - EDL 1.0 - The Go license - Bouncy Castle Licence - - test,provided,system - true - - Apache License 2.0 - |The Apache License, Version 2.0 - |The Apache Software License, Version 2.0 - |Apache-2.0 - |Apache License, Version 2.0 - |Apache 2.0 - |Apache License V2.0 - BSD 2-Clause License - |The BSD License - The MIT License|MIT License - 3-Clause BSD License|BSD-3-Clause - - - - - - - - checkLinkageErrors - - - - maven-enforcer-plugin - - - enforce-linkage-checker - none - - enforce - - - - - true - ${basedir}/linkage-checker-exclusion-rules.xml - WARN - - - - - - - - - - - - - - checkShadedContent - - - - org.codehaus.mojo - exec-maven-plugin - - - check-shaded-content - verify - - exec - - - ${basedir}/scripts/check_content.sh - - - - - - - - - shadeDep - - - - maven-dependency-plugin - - - generate-resources - - list - - - runtime - ${license.processing.dependencyListFile} - - - - copy-dependencies - generate-resources - - copy-dependencies - - - ${license.processing.dependencyJarsDir} - false - false - true - - - - - - maven-shade-plugin - 3.5.2 - - - package - - shade - - - - - - - net.snowflake:snowflake-jdbc - org.slf4j:slf4j-api - com.github.luben:zstd-jni - - - - - com.nimbusds - ${shadeBase}.com.nimbusds - - - org.bouncycastle - ${shadeBase}.org.bouncycastle - - - net.jcip - ${shadeBase}.net.jcip - - - net.minidev - ${shadeBase}.net.minidev - - - org.objectweb - ${shadeBase}.org.objectweb - - - com.fasterxml - ${shadeBase}.fasterxml - - - org.apache - ${shadeBase}.apache - - - org.xbill - ${shadeBase}.org.xbill - - - org.xerial - ${shadeBase}.org.xerial - - - io.netty - ${shadeBase}.io.netty - - - com.google - ${shadeBase}.com.google - - - com.github.benmanes - ${shadeBase}.com.github.benmanes - - - shaded.parquet - ${shadeBase}.shaded.parquet - - - org.codehaus - ${shadeBase}.org.codehaus - - - com.jcraft - ${shadeBase}.com.jcraft - - - org.eclipse - ${shadeBase}.org.eclipse - - - org.checkerframework - ${shadeBase}.org.checkerframework - - - com.codahale - ${shadeBase}.com.codahale - - - com.ctc - ${shadeBase}.com.ctc - - - com.thoughtworks - ${shadeBase}.com.thoughtworks - - - codegen - ${shadeBase}.codegen - - - javax.annotation - ${shadeBase}.javax.annotation - - - javax.activation - ${shadeBase}.javax.activation - - - io.airlift.compress - ${shadeBase}.io.airlift.compress - - - - - *:* - - assets/org/apache/commons/math3/exception/util/LocalizedFormats_fr.properties - about.html - mozilla/public-suffix-list.txt - META-INF/LICENSE* - META-INF/NOTICE* - META-INF/DEPENDENCIES - META-INF/maven/** - META-INF/services/com.fasterxml.* - META-INF/*.xml - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - google/protobuf/**/*.proto - - - - commons-logging:commons-logging - - org/apache/commons/logging/impl/AvalonLogger.class - - - - org.slf4j:slf4j-simple - - org/slf4j/** - - - - - - - - - - - org.codehaus.mojo - exec-maven-plugin - - - process-third-party-licenses - generate-resources - - exec - - - python3 - - ${project.basedir}/scripts/process_licenses.py - ${license.processing.dependencyListFile} - ${license.processing.dependencyJarsDir} - ${license.processing.targetDir} - - - - - - - - - - ossrh-deploy - - - - maven-gpg-plugin - 1.6 - - - deploy - - sign-and-deploy-file - - - target/${project.artifactId}.jar - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2 - generated_public_pom.xml - target/${project.artifactId}-javadoc.jar - target/${project.artifactId}-sources.jar - ${env.GPG_KEY_ID} - ${env.GPG_KEY_PASSPHRASE} - - - - - - - - - snapshot-deploy - - - - maven-gpg-plugin - 1.6 - - - deploy - - sign-and-deploy-file - - - target/${project.artifactId}.jar - snapshot - https://nexus.int.snowflakecomputing.com/repository/Snapshots/ - generated_public_pom.xml - target/${project.artifactId}-javadoc.jar - target/${project.artifactId}-sources.jar - ${env.GPG_KEY_ID} - ${env.GPG_KEY_PASSPHRASE} - - - - - - - - - ghActionsIT - - - - maven-failsafe-plugin - - - - integration-test - - - - verify_github_actions_it - verify - - verify - - - - - - org.jacoco - jacoco-maven-plugin - ${jacoco.version} - - - pre-unit-test - - prepare-agent - - - target/jacoco-ut.exec - - - - post-unit-test - test - - report - - - target/jacoco-ut.exec - target/jacoco-ut - - - - pre-integration-test - pre-integration-test - - prepare-agent - - - target/jacoco-it.exec - - - - post-integration-test - post-integration-test - - report - - - target/jacoco-it.exec - target/jacoco-it - - - - - ${jacoco.skip.instrument} - - - - - - - - - net.bytebuddy - byte-buddy-agent - 1.14.9 - test - - - net.snowflake - snowflake-jdbc - 3.16.1 - compile - - - org.slf4j - slf4j-api - 1.7.36 - compile - - - com.github.luben - zstd-jni - 1.5.6-2 - runtime - - - junit - junit - 4.13.2 - test - - - org.apache.commons - commons-lang3 - 3.14.0 - test - - - org.hamcrest - hamcrest-core - 1.3 - test - - - org.mockito - mockito-core - 3.7.7 - test - - - byte-buddy - net.bytebuddy - - - objenesis - org.objenesis - - - - - org.openjdk.jmh - jmh-core - 1.34 - test - - - jopt-simple - net.sf.jopt-simple - - - - - org.openjdk.jmh - jmh-generator-annprocess - 1.34 - test - - - org.powermock - powermock-api-mockito2 - 2.0.9 - test - - - powermock-api-support - org.powermock - - - - - org.powermock - powermock-core - 2.0.9 - test - - - powermock-reflect - org.powermock - - - javassist - org.javassist - - - byte-buddy - net.bytebuddy - - - - - org.powermock - powermock-module-junit4 - 2.0.9 - test - - - powermock-module-junit4-common - org.powermock - - - - - org.slf4j - slf4j-simple - 1.7.36 - test - - - - - - com.fasterxml.jackson - jackson-bom - ${fasterxml.version} - pom - import - - - com.google.guava - guava - ${guava.version} - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - com.nimbusds - nimbus-jose-jwt - ${nimbusds.version} - - - commons-codec - commons-codec - ${commonscodec.version} - - - commons-collections - commons-collections - ${commonscollections.version} - - - commons-io - commons-io - ${commonsio.version} - - - commons-logging - commons-logging - ${commonslogging.version} - - - io.netty - netty-buffer - ${netty.version} - - - io.netty - netty-common - ${netty.version} - - - net.minidev - json-smart - ${net.minidev.version} - - - net.snowflake - snowflake-jdbc - ${snowjdbc.version} - - - org.apache.commons - commons-compress - ${commonscompress.version} - - - org.apache.commons - commons-configuration2 - ${commonsconfiguration2.version} - - - org.apache.commons - commons-lang3 - ${commonslang3.version} - - - org.apache.commons - commons-text - ${commonstext.version} - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - - - reload4j - ch.qos.reload4j - - - jersey-json - com.github.pjfanning - - - jsch - com.jcraft - - - jersey-core - com.sun.jersey - - - jersey-server - com.sun.jersey - - - jersey-servlet - com.sun.jersey - - - commons-logging - commons-logging - - - dnsjava - dnsjava - - - metrics-core - io.dropwizard.metrics - - - activation - javax.activation - - - javax.servlet-api - javax.servlet - - - jsp-api - javax.servlet.jsp - - - avro - org.apache.avro - - - curator-client - org.apache.curator - - - curator-recipes - org.apache.curator - - - hadoop-auth - org.apache.hadoop - - - httpclient - org.apache.httpcomponents - - - kerb-core - org.apache.kerby - - - zookeeper - org.apache.zookeeper - - - jetty-server - org.eclipse.jetty - - - jetty-servlet - org.eclipse.jetty - - - jetty-util - org.eclipse.jetty - - - jetty-webapp - org.eclipse.jetty - - - slf4j-log4j12 - org.slf4j - - - slf4j-reload4j - org.slf4j - - - - - org.apache.parquet - parquet-column - ${parquet.version} - - - org.apache.parquet - parquet-common - ${parquet.version} - - - org.apache.parquet - parquet-hadoop - ${parquet.version} - - - org.apache.yetus - audience-annotations - ${yetus.version} - - - org.bouncycastle - bcpkix-jdk18on - ${bouncycastle.version} - - - org.bouncycastle - bcprov-jdk18on - ${bouncycastle.version} - - - org.codehaus.jackson - jackson-core-asl - ${codehaus.version} - - - org.codehaus.jackson - jackson-jaxrs - ${codehaus.version} - - - org.codehaus.jackson - jackson-xc - ${codehaus.version} - - - org.codehaus.jettison - jettison - 1.5.4 - - - org.codehaus.woodstox - stax2-api - 4.2.1 - - - org.objenesis - objenesis - ${objenesis.version} - - - org.slf4j - slf4j-api - ${slf4j.version} - - - org.xerial.snappy - snappy-java - ${snappy.version} - - - junit - junit - 4.13.2 - test - - - net.bytebuddy - byte-buddy - 1.14.9 - test - - - net.bytebuddy - byte-buddy-agent - 1.14.9 - test - - - org.mockito - mockito-core - 3.7.7 - test - - - org.openjdk.jmh - jmh-core - 1.34 - test - - - org.openjdk.jmh - jmh-generator-annprocess - 1.34 - test - - - - - 3.14.0 - 1.11.0 - 1.1.10.5 - 2.10.1 - 3.3.6 - 1.7.36 - ${project.build.directory}/dependency-list.txt - 4.27.2 - 2.15.1 - ${project.build.directory}/generated-licenses-info - 1.3.1 - 1.78.1 - 1.8 - 0.8.5 - 3.16.1 - 9.37.3 - 0.13.0 - 2.4.9 - 1.9.13 - 4.1.94.Final - true - 2.17.0 - 32.0.1-jre - ${project.build.directory}/dependency-jars - 2.0.9 - UTF-8 - net.snowflake.ingest.internal - 3.1 - 1.14.1 - ${license.processing.resourcesRoot}/META-INF/third-party-licenses - 1.15 - 1.26.0 - 3.2.2 - 1.8 - - diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java index 599f92938..ec1608b3e 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java @@ -110,7 +110,7 @@ public class SnowflakeStreamingIngestClientInternal implements SnowflakeStrea private final FlushService flushService; // Reference to storage manager - private final IStorageManager storageManager; + private IStorageManager storageManager; // Indicates whether the client has closed private volatile boolean isClosed; @@ -1077,4 +1077,9 @@ private void cleanUpResources() { public Map getEncryptionKeysPerTable() { return encryptionKeysPerTable; } + + // TESTING ONLY - inject the storage manager + public void setStorageManager(IStorageManager storageManager) { + this.storageManager = (IStorageManager) storageManager; + } } diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java index 593f09021..e0a1703d6 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java @@ -1588,6 +1588,14 @@ public void testOpenChannelWithEncryptionKey() throws Exception { CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class); StatusLine statusLine = Mockito.mock(StatusLine.class); HttpEntity httpEntity = Mockito.mock(HttpEntity.class); + IStorageManager storageManager = + Mockito.spy( + isIcebergMode + ? new ExternalVolumeManager<>(true, "role", "client", null) + : new InternalStageManager<>(true, "role", "client", null)); + Mockito.doNothing() + .when(storageManager) + .addStorage(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); when(statusLine.getStatusCode()).thenReturn(200); when(httpResponse.getStatusLine()).thenReturn(statusLine); when(httpResponse.getEntity()).thenReturn(httpEntity); @@ -1621,6 +1629,7 @@ public void testOpenChannelWithEncryptionKey() throws Exception { true, requestBuilder, null); + client.setStorageManager(storageManager); OpenChannelRequest request = OpenChannelRequest.builder("channel")