Removes some unnecessary dependencies in the S3 sink and Parquet code…

…cs (opensearch-project#3275) Removes some unnecessary dependencies in the S3 sink and Parquet codec projects. Updating the Parquet version to 1.13.1 consistently. Exclude HDFS client. Signed-off-by: David Venable <[email protected]>
engechas · Aug 29, 2023 · 005f2aa · 005f2aa
1 parent 14ec2e4
commit 005f2aa
Show file tree

Hide file tree

Showing 8 changed files with 19 additions and 31 deletions.
diff --git a/data-prepper-api/build.gradle b/data-prepper-api/build.gradle
@@ -8,7 +8,7 @@ dependencies {
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.15.2'
-    implementation 'org.apache.parquet:parquet-common:1.12.3'
+    implementation 'org.apache.parquet:parquet-common:1.13.1'
     testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
     implementation libs.commons.lang3
     testImplementation testLibs.junit.vintage

diff --git a/data-prepper-plugins/avro-codecs/build.gradle b/data-prepper-plugins/avro-codecs/build.gradle
@@ -6,7 +6,7 @@
 dependencies {
     implementation project(path: ':data-prepper-api')
     implementation 'org.apache.avro:avro:1.11.1'
-    implementation 'org.apache.parquet:parquet-common:1.12.3'
+    implementation 'org.apache.parquet:parquet-common:1.13.1'
     implementation 'software.amazon.awssdk:s3'
     implementation 'software.amazon.awssdk:apache-client'
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.2'

diff --git a/data-prepper-plugins/common/build.gradle b/data-prepper-plugins/common/build.gradle
@@ -20,7 +20,7 @@ dependencies {
     implementation 'org.reflections:reflections:0.10.2'
     implementation 'io.micrometer:micrometer-core'
     testImplementation testLibs.junit.vintage
-    implementation 'org.apache.parquet:parquet-common:1.12.3'
+    implementation 'org.apache.parquet:parquet-common:1.13.1'
     implementation 'org.xerial.snappy:snappy-java:1.1.10.3'
     testImplementation project(':data-prepper-plugins:blocking-buffer')
     testImplementation 'commons-io:commons-io:2.12.0'

diff --git a/data-prepper-plugins/csv-processor/build.gradle b/data-prepper-plugins/csv-processor/build.gradle
@@ -12,7 +12,7 @@ dependencies {
     implementation project(':data-prepper-api')
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv'
     implementation 'io.micrometer:micrometer-core'
-    implementation 'org.apache.parquet:parquet-common:1.12.3'
+    implementation 'org.apache.parquet:parquet-common:1.13.1'
     implementation 'software.amazon.awssdk:s3'
     implementation 'software.amazon.awssdk:apache-client'
     testImplementation project(':data-prepper-plugins:log-generator-source')

diff --git a/data-prepper-plugins/newline-codecs/build.gradle b/data-prepper-plugins/newline-codecs/build.gradle
@@ -9,11 +9,8 @@ repositories {
 
 dependencies {
     implementation project(':data-prepper-api')
-    implementation 'com.fasterxml.jackson.core:jackson-core'
-    implementation 'com.fasterxml.jackson.core:jackson-databind'
-    implementation 'org.apache.parquet:parquet-common:1.12.3'
-    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1'
-    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1'
+    implementation 'com.fasterxml.jackson.core:jackson-annotations'
+    implementation 'org.apache.parquet:parquet-common:1.13.1'
     testImplementation project(':data-prepper-plugins:common')
 }
 

diff --git a/data-prepper-plugins/parquet-codecs/build.gradle b/data-prepper-plugins/parquet-codecs/build.gradle
@@ -7,19 +7,14 @@ dependencies {
     implementation project(':data-prepper-api')
     implementation project(':data-prepper-plugins:common')
     implementation 'org.apache.avro:avro:1.11.0'
-    implementation 'software.amazon.awssdk:s3'
-    implementation 'software.amazon.awssdk:apache-client'
     implementation 'org.apache.hadoop:hadoop-common:3.3.5'
-    implementation 'org.apache.hadoop:hadoop-hdfs-client:3.3.5'
-    implementation 'org.apache.hadoop:hadoop-yarn-client:3.3.5'
-    implementation 'org.apache.hadoop:hadoop-yarn-common:3.3.5'
-    implementation 'org.apache.hadoop:hadoop-mapreduce-client-core:3.3.5'
-    implementation 'org.apache.parquet:parquet-avro:1.12.3'
-    implementation 'org.apache.parquet:parquet-column:1.12.3'
-    implementation 'org.apache.parquet:parquet-common:1.12.3'
-    implementation 'org.apache.parquet:parquet-hadoop:1.12.3'
-    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1'
-    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1'
+    implementation('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.5') {
+        exclude group: 'org.apache.hadoop', module: 'hadoop-hdfs-client'
+    }
+    implementation 'org.apache.parquet:parquet-avro:1.13.1'
+    implementation 'org.apache.parquet:parquet-column:1.13.1'
+    implementation 'org.apache.parquet:parquet-common:1.13.1'
+    implementation 'org.apache.parquet:parquet-hadoop:1.13.1'
     testImplementation project(':data-prepper-test-common')
 }
 

diff --git a/data-prepper-plugins/parse-json-processor/build.gradle b/data-prepper-plugins/parse-json-processor/build.gradle
@@ -15,7 +15,7 @@ dependencies {
     implementation project(':data-prepper-api')
     implementation project(':data-prepper-plugins:common')
     implementation 'com.fasterxml.jackson.core:jackson-databind'
-    implementation 'org.apache.parquet:parquet-common:1.12.3'
+    implementation 'org.apache.parquet:parquet-common:1.13.1'
     testImplementation project(':data-prepper-test-common')
 }
 

diff --git a/data-prepper-plugins/s3-source/build.gradle b/data-prepper-plugins/s3-source/build.gradle
@@ -31,7 +31,7 @@ dependencies {
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv'
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2'
     implementation 'org.xerial.snappy:snappy-java:1.1.10.1'
-    implementation 'org.apache.parquet:parquet-common:1.12.3'
+    implementation 'org.apache.parquet:parquet-common:1.13.1'
     implementation 'dev.failsafe:failsafe:3.3.2'
     implementation 'org.apache.httpcomponents:httpcore:4.4.15'
     testImplementation libs.commons.lang3
@@ -48,14 +48,10 @@ dependencies {
     testImplementation project(':data-prepper-plugins:parquet-codecs')
     testImplementation 'org.apache.avro:avro:1.11.0'
     testImplementation 'org.apache.hadoop:hadoop-common:3.3.5'
-    testImplementation 'org.apache.hadoop:hadoop-hdfs-client:3.3.5'
-    testImplementation 'org.apache.hadoop:hadoop-yarn-client:3.3.5'
-    testImplementation 'org.apache.hadoop:hadoop-yarn-common:3.3.5'
-    testImplementation 'org.apache.hadoop:hadoop-mapreduce-client-core:3.3.5'
-    testImplementation 'org.apache.parquet:parquet-avro:1.12.3'
-    testImplementation 'org.apache.parquet:parquet-column:1.12.3'
-    testImplementation 'org.apache.parquet:parquet-common:1.12.3'
-    testImplementation 'org.apache.parquet:parquet-hadoop:1.12.3'
+    testImplementation 'org.apache.parquet:parquet-avro:1.13.1'
+    testImplementation 'org.apache.parquet:parquet-column:1.13.1'
+    testImplementation 'org.apache.parquet:parquet-common:1.13.1'
+    testImplementation 'org.apache.parquet:parquet-hadoop:1.13.1'
     constraints {
         testImplementation('org.eclipse.jetty:jetty-bom') {
             version {