SNOW-1707031 hadoop upgrade for vulnerability fix (#849)

* fix: pom.xml to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-ORGAPACHEHADOOP-8089372 * test * test remove * revert * fix whitespace * upgrade netty too for the hadoop upgrade * Fix maven package * Fix maven install - works locally --------- Co-authored-by: snyk-bot <[email protected]> Co-authored-by: Xin Huang <[email protected]>
snowflakedb · Oct 3, 2024 · ef92ea9 · ef92ea9
1 parent 0930648
commit ef92ea9
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 2 deletions.
diff --git a/pom.xml b/pom.xml
@@ -50,7 +50,7 @@
     <commonstext.version>1.11.0</commonstext.version>
     <fasterxml.version>2.17.2</fasterxml.version>
     <guava.version>32.0.1-jre</guava.version>
-    <hadoop.version>3.3.6</hadoop.version>
+    <hadoop.version>3.4.0</hadoop.version>
     <iceberg.version>1.5.2</iceberg.version>
     <jacoco.skip.instrument>true</jacoco.skip.instrument>
     <jacoco.version>0.8.5</jacoco.version>
@@ -61,7 +61,7 @@
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
     <net.minidev.version>2.4.9</net.minidev.version>
-    <netty.version>4.1.94.Final</netty.version>
+    <netty.version>4.1.113.Final</netty.version>
     <nimbusds.version>9.37.3</nimbusds.version>
     <objenesis.version>3.1</objenesis.version>
     <parquet.version>1.14.1</parquet.version>
@@ -240,6 +240,10 @@
             <groupId>org.apache.zookeeper</groupId>
             <artifactId>zookeeper</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcprov-jdk15on</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>org.eclipse.jetty</groupId>
             <artifactId>jetty-server</artifactId>
@@ -385,6 +389,14 @@
             <groupId>javax.xml.bind</groupId>
             <artifactId>jaxb-api</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-common</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcprov-jdk15on</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-reload4j</artifactId>

diff --git a/scripts/check_content.sh b/scripts/check_content.sh
@@ -28,6 +28,7 @@ if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | \
     grep -v PropertyList-1.0.dtd | \
     grep -v properties.dtd | \
     grep -v parquet.thrift | \
+    grep -v assets/org/apache/commons/math3/random/new-joe-kuo-6.1000 | \
 
     # Native zstd libraries are allowed
     grep -v -E '^darwin' | \

diff --git a/scripts/process_licenses.py b/scripts/process_licenses.py
@@ -50,6 +50,14 @@
     "com.nimbusds:nimbus-jose-jwt": APACHE_LICENSE,
     "com.github.stephenc.jcip:jcip-annotations": APACHE_LICENSE,
     "io.netty:netty-common": APACHE_LICENSE,
+    "io.netty:netty-handler": APACHE_LICENSE,
+    "io.netty:netty-resolver": APACHE_LICENSE,
+    "io.netty:netty-buffer": APACHE_LICENSE,
+    "io.netty:netty-transport": APACHE_LICENSE,
+    "io.netty:netty-transport-native-unix-common": APACHE_LICENSE,
+    "io.netty:netty-codec": APACHE_LICENSE,
+    "io.netty:netty-transport-native-epoll": APACHE_LICENSE,
+    "io.netty:netty-transport-classes-epoll": APACHE_LICENSE,
     "com.google.re2j:re2j": GO_LICENSE,
     "com.google.protobuf:protobuf-java": BSD_3_CLAUSE_LICENSE,
     "com.google.code.gson:gson": APACHE_LICENSE,