From 75a07273576b2d08814e15ab90cd7e2d00f363da Mon Sep 17 00:00:00 2001 From: Toby Zhang Date: Sat, 15 Jun 2024 00:14:41 +0000 Subject: [PATCH 1/5] fix cve --- pom.xml | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index e14af3d8e..04062e6a8 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,7 @@ + 0.27 1.78.1 1.9.13 1.15 @@ -115,6 +116,11 @@ commons-logging ${commonslogging.version} + + io.airlift + aircompressor + ${aircompressor.version} + io.netty netty-buffer @@ -414,6 +420,10 @@ commons-codec commons-codec + + io.airlift + aircompressor + @@ -723,10 +733,11 @@ true + to workaround https://issues.apache.org/jira/browse/MNG-7982. Now the dependency analyzer complains that + the dependency is unused, so we ignore it here--> org.apache.commons:commons-compress org.apache.commons:commons-configuration2 + io.airlift:aircompressor @@ -819,9 +830,9 @@ failFast + The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please + check your dependencies", verify the conditions of the license and add the reference to it here. + --> Apache License 2.0 BSD 2-Clause License @@ -1133,9 +1144,9 @@ + Plugin executes license processing Python script, which copies third party license files into the directory + target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR. + --> org.codehaus.mojo exec-maven-plugin From f2c4e0bf0d613dd8e3ab2f24bb79d79fd43441bb Mon Sep 17 00:00:00 2001 From: Toby Zhang Date: Thu, 20 Jun 2024 23:09:25 +0000 Subject: [PATCH 2/5] Revert "SNOW-1457523: Fix CVE for snowflake-ingest-java io.airlift:aircompressor 0.21 (#774)" This reverts commit 1357e7499ef082805f7ac682b433ef0e863a2600. --- pom.xml | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index 04062e6a8..e14af3d8e 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,6 @@ - 0.27 1.78.1 1.9.13 1.15 @@ -116,11 +115,6 @@ commons-logging ${commonslogging.version} - - io.airlift - aircompressor - ${aircompressor.version} - io.netty netty-buffer @@ -420,10 +414,6 @@ commons-codec commons-codec - - io.airlift - aircompressor - @@ -733,11 +723,10 @@ true + to workaround https://issues.apache.org/jira/browse/MNG-7982. Now the dependency analyzer complains that + the dependency is unused, so we ignore it here--> org.apache.commons:commons-compress org.apache.commons:commons-configuration2 - io.airlift:aircompressor @@ -830,9 +819,9 @@ failFast + The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please + check your dependencies", verify the conditions of the license and add the reference to it here. + --> Apache License 2.0 BSD 2-Clause License @@ -1144,9 +1133,9 @@ + Plugin executes license processing Python script, which copies third party license files into the directory + target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR. + --> org.codehaus.mojo exec-maven-plugin From e0334c6c766f54e29e554bd1ae6dd8be47665a75 Mon Sep 17 00:00:00 2001 From: Toby Zhang Date: Thu, 20 Jun 2024 23:34:55 +0000 Subject: [PATCH 3/5] update pom --- pom.xml | 29 +++++++++++++++-------------- scripts/process_licenses.py | 9 +++++++-- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/pom.xml b/pom.xml index e14af3d8e..4e6907047 100644 --- a/pom.xml +++ b/pom.xml @@ -45,7 +45,7 @@ 3.14.0 1.3.1 1.11.0 - 2.16.1 + 2.17.0 32.0.1-jre 3.3.6 true @@ -60,13 +60,13 @@ 4.1.94.Final 9.37.3 3.1 - 1.13.1 + 1.14.1 2.0.9 UTF-8 3.19.6 net.snowflake.ingest.internal 1.7.36 - 1.1.10.4 + 1.1.10.5 3.16.1 0.13.0 @@ -343,13 +343,13 @@ net.bytebuddy byte-buddy - 1.10.19 + 1.14.9 test net.bytebuddy byte-buddy-agent - 1.10.19 + 1.14.9 test @@ -491,7 +491,7 @@ com.github.luben zstd-jni - 1.5.0-1 + 1.5.6-2 runtime @@ -723,8 +723,8 @@ true + to workaround https://issues.apache.org/jira/browse/MNG-7982. Now the dependency analyzer complains that + the dependency is unused, so we ignore it here--> org.apache.commons:commons-compress org.apache.commons:commons-configuration2 @@ -819,9 +819,9 @@ failFast + The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please + check your dependencies", verify the conditions of the license and add the reference to it here. + --> Apache License 2.0 BSD 2-Clause License @@ -830,6 +830,7 @@ EDL 1.0 The Go license Bouncy Castle Licence + CDDL + GPLv2 with classpath exception test,provided,system true @@ -1133,9 +1134,9 @@ + Plugin executes license processing Python script, which copies third party license files into the directory + target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR. + --> org.codehaus.mojo exec-maven-plugin diff --git a/scripts/process_licenses.py b/scripts/process_licenses.py index bb43fbbf0..0e525efca 100644 --- a/scripts/process_licenses.py +++ b/scripts/process_licenses.py @@ -31,6 +31,7 @@ MIT_LICENSE = "The MIT License" GO_LICENSE = "The Go license" BOUNCY_CASTLE_LICENSE = "Bouncy Castle Licence " +CDDL_GPLv2 = "CDDL + GPLv2 with classpath exception" # The SDK does not need to include licenses of dependencies, which aren't shaded IGNORED_DEPENDENCIES = {"net.snowflake:snowflake-jdbc", "org.slf4j:slf4j-api"} @@ -61,6 +62,7 @@ "org.bouncycastle:bcpkix-jdk18on": BOUNCY_CASTLE_LICENSE, "org.bouncycastle:bcutil-jdk18on": BOUNCY_CASTLE_LICENSE, "org.bouncycastle:bcprov-jdk18on": BOUNCY_CASTLE_LICENSE, + "javax.annotation:javax.annotation-api": CDDL_GPLv2 } @@ -132,18 +134,21 @@ def main(): dependency_without_license_count += 1 missing_licenses_str += f"{dependency_lookup_key}: {license_name}\n" else: - raise Exception(f"The dependency {dependency_lookup_key} does not ship a license file, but neither is it not defined in ADDITIONAL_LICENSES_MAP") + raise Exception( + f"The dependency {dependency_lookup_key} does not ship a license file, but neither is it not defined in ADDITIONAL_LICENSES_MAP") with open(Path(target_dir, "ADDITIONAL_LICENCES"), "w") as additional_licenses_handle: additional_licenses_handle.write(missing_licenses_str) if dependency_count < 30: - raise Exception(f"Suspiciously low number of dependency JARs detected in {dependency_jars_path}: {dependency_count}") + raise Exception( + f"Suspiciously low number of dependency JARs detected in {dependency_jars_path}: {dependency_count}") print("License generation finished") print(f"\tTotal dependencies: {dependency_count}") print(f"\tTotal dependencies (with license): {dependency_with_license_count}") print(f"\tTotal dependencies (without license): {dependency_without_license_count}") print(f"\tIgnored dependencies: {dependency_ignored_count}") + if __name__ == "__main__": main() From 8bddd2c304fcc97424a45792592af29ddfeec244 Mon Sep 17 00:00:00 2001 From: Toby Zhang Date: Fri, 21 Jun 2024 01:40:19 +0000 Subject: [PATCH 4/5] refresh token --- .../internal/StreamingIngestStage.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java index e8e56f383..ed73e3774 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java @@ -17,6 +17,8 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Paths; +import java.time.Duration; +import java.time.Instant; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -47,6 +49,11 @@ class StreamingIngestStage { private static final long REFRESH_THRESHOLD_IN_MS = TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES); + // Stage credential refresh interval, currently the token will expire in 1hr for GCS and 2hr for + // AWS/Azure, so set it a bit smaller than 1hr + private static final Duration refreshDuration = Duration.ofMinutes(58); + private static Instant prevRefresh = Instant.EPOCH; + private static final Logging logger = new Logging(StreamingIngestStage.class); /** @@ -180,6 +187,12 @@ private void putRemote(String fullFilePath, byte[] data, int retryCount) InputStream inStream = new ByteArrayInputStream(data); try { + // Proactively refresh the credential if it's going to expire, to avoid the token expiration + // error from JDBC which confuses customer + if (Instant.now().isAfter(prevRefresh.plus(refreshDuration))) { + refreshSnowflakeMetadata(); + } + SnowflakeFileTransferAgent.uploadWithoutConnection( SnowflakeFileTransferConfig.Builder.newInstance() .setSnowflakeFileTransferMetadata(fileTransferMetadataCopy) @@ -194,9 +207,6 @@ private void putRemote(String fullFilePath, byte[] data, int retryCount) } catch (Exception e) { if (retryCount == 0) { // for the first exception, we always perform a metadata refresh. - logger.logInfo( - "Stage metadata need to be refreshed due to upload error: {} on first retry attempt", - e.getMessage()); this.refreshSnowflakeMetadata(); } if (retryCount >= maxUploadRetries) { @@ -281,6 +291,8 @@ synchronized SnowflakeFileTransferMetadataWithAge refreshSnowflakeMetadata(boole SnowflakeFileTransferAgent.getFileTransferMetadatas(responseNode).get(0), Optional.of(System.currentTimeMillis())); } + + prevRefresh = Instant.now(); return this.fileTransferMetadataWithAge; } From 7eadb4aa9d60b97f4aa5f7cd96f0dcc9e608c4da Mon Sep 17 00:00:00 2001 From: Toby Zhang Date: Sat, 13 Jul 2024 01:17:30 +0000 Subject: [PATCH 5/5] fix pom --- pom.xml | 41 +++++++++-------------------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/pom.xml b/pom.xml index c81655628..4709f39bf 100644 --- a/pom.xml +++ b/pom.xml @@ -483,6 +483,7 @@ org.apache.parquet parquet-common + javax.annotation javax.annotation-api @@ -751,15 +752,8 @@ true - >>>>>>> master -======= - to workaround https://issues.apache.org/jira/browse/MNG-7982. Now the dependency analyzer complains that - the dependency is unused, so we ignore it here--> ->>>>>>> e6c4413ec391df44c2ec2ff62e59cc96b3f32ce2 + to workaround https://issues.apache.org/jira/browse/MNG-7982. Now the dependency analyzer complains that + the dependency is unused, so we ignore it here--> org.apache.commons:commons-compress org.apache.commons:commons-configuration2 @@ -853,22 +847,10 @@ 2.0.1 failFast -<<<<<<< HEAD - ======= - The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses - used, please - check your dependencies", verify the conditions of the license and add the reference to it here. - --> - >>>>>>> master -======= - ->>>>>>> e6c4413ec391df44c2ec2ff62e59cc96b3f32ce2 + The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please + check your dependencies", verify the conditions of the license and add the reference to it here. + --> Apache License 2.0 BSD 2-Clause License @@ -1179,15 +1161,10 @@ -<<<<<<< HEAD -======= - ->>>>>>> e6c4413ec391df44c2ec2ff62e59cc96b3f32ce2 + Plugin executes license processing Python script, which copies third party license files into the directory + target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR. + --> org.codehaus.mojo exec-maven-plugin