Updates Parquet to 1.14.1 and Hadoop to 3.4.0. Make use of Gradle's v…

…ersion catalogue for Parquet. (opensearch-project#4705)

Signed-off-by: David Venable <[email protected]>

data-prepper-api/build.gradle

@@ -12,7 +12,7 @@ dependencies {
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8'
-    implementation 'org.apache.parquet:parquet-common:1.14.0'
+    implementation libs.parquet.common
     testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
     implementation libs.commons.lang3
     testImplementation project(':data-prepper-test-common')

data-prepper-plugins/avro-codecs/build.gradle

@@ -6,7 +6,7 @@
 dependencies {
     implementation project(path: ':data-prepper-api')
     implementation libs.avro.core
-    implementation 'org.apache.parquet:parquet-common:1.14.0'
+    implementation libs.parquet.common
     implementation 'software.amazon.awssdk:s3'
     implementation 'software.amazon.awssdk:apache-client'
     testImplementation 'org.json:json:20240205'

data-prepper-plugins/common/build.gradle

@@ -19,7 +19,7 @@ dependencies {
     implementation libs.bouncycastle.bcpkix
     implementation libs.reflections.core
     implementation 'io.micrometer:micrometer-core'
-    implementation 'org.apache.parquet:parquet-common:1.14.0'
+    implementation libs.parquet.common
     implementation 'org.xerial.snappy:snappy-java:1.1.10.5'
     testImplementation project(':data-prepper-plugins:blocking-buffer')
     testImplementation project(':data-prepper-test-event')

data-prepper-plugins/csv-processor/build.gradle

@@ -12,7 +12,7 @@ dependencies {
     implementation project(':data-prepper-api')
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv'
     implementation 'io.micrometer:micrometer-core'
-    implementation 'org.apache.parquet:parquet-common:1.14.0'
+    implementation libs.parquet.common
     implementation 'software.amazon.awssdk:s3'
     implementation 'software.amazon.awssdk:apache-client'
     testImplementation project(':data-prepper-plugins:log-generator-source')

data-prepper-plugins/event-json-codecs/build.gradle

@@ -15,7 +15,7 @@ dependencies {
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml'
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.17.0'
     testImplementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.17.0'
-    implementation 'org.apache.parquet:parquet-common:1.14.0'
+    implementation libs.parquet.common
     testImplementation project(':data-prepper-test-common')
 }
 

data-prepper-plugins/newline-codecs/build.gradle

@@ -5,7 +5,7 @@ plugins {
 dependencies {
     implementation project(':data-prepper-api')
     implementation 'com.fasterxml.jackson.core:jackson-annotations'
-    implementation 'org.apache.parquet:parquet-common:1.14.0'
+    implementation libs.parquet.common
     testImplementation project(':data-prepper-plugins:common')
     testImplementation project(':data-prepper-test-event')
 }

data-prepper-plugins/parquet-codecs/build.gradle

@@ -8,10 +8,10 @@ dependencies {
     implementation project(':data-prepper-plugins:common')
     implementation libs.avro.core
     implementation 'org.apache.commons:commons-text:1.11.0'
-    implementation 'org.apache.parquet:parquet-avro:1.14.0'
-    implementation 'org.apache.parquet:parquet-column:1.14.0'
-    implementation 'org.apache.parquet:parquet-common:1.14.0'
-    implementation 'org.apache.parquet:parquet-hadoop:1.14.0'
+    implementation libs.parquet.avro
+    implementation libs.parquet.column
+    implementation libs.parquet.common
+    implementation libs.parquet.hadoop
     runtimeOnly(libs.hadoop.common) {
         exclude group: 'org.eclipse.jetty'
         exclude group: 'org.apache.hadoop', module: 'hadoop-auth'

data-prepper-plugins/parse-json-processor/build.gradle

@@ -13,7 +13,7 @@ dependencies {
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-ion'
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml'
-    implementation 'org.apache.parquet:parquet-common:1.14.0'
+    implementation libs.parquet.common
     testImplementation project(':data-prepper-test-common')
     testImplementation project(':data-prepper-test-event')
 }

data-prepper-plugins/s3-sink/build.gradle

@@ -23,7 +23,7 @@ dependencies {
         exclude group: 'org.eclipse.jetty'
         exclude group: 'org.apache.hadoop', module: 'hadoop-auth'
     }
-    implementation 'org.apache.parquet:parquet-avro:1.14.0'
+    implementation libs.parquet.avro
     implementation 'software.amazon.awssdk:apache-client'
     implementation 'org.jetbrains.kotlin:kotlin-stdlib-common:1.9.22'
     implementation libs.commons.lang3

data-prepper-plugins/s3-source/build.gradle

@@ -27,7 +27,7 @@ dependencies {
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv'
     implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
     implementation 'org.xerial.snappy:snappy-java:1.1.10.5'
-    implementation 'org.apache.parquet:parquet-common:1.14.0'
+    implementation libs.parquet.common
     implementation 'dev.failsafe:failsafe:3.3.2'
     implementation 'org.apache.httpcomponents:httpcore:4.4.16'
     testImplementation libs.commons.lang3
@@ -45,11 +45,10 @@ dependencies {
     testImplementation project(':data-prepper-plugins:parquet-codecs')
     testImplementation project(':data-prepper-test-event')
     testImplementation libs.avro.core
-    testImplementation testLibs.hadoop.common
-    testImplementation 'org.apache.parquet:parquet-avro:1.14.0'
-    testImplementation 'org.apache.parquet:parquet-column:1.14.0'
-    testImplementation 'org.apache.parquet:parquet-common:1.14.0'
-    testImplementation 'org.apache.parquet:parquet-hadoop:1.14.0'
+    testImplementation libs.hadoop.common
+    testImplementation libs.parquet.avro
+    testImplementation libs.parquet.column
+    testImplementation libs.parquet.hadoop
 }
 
 test {

settings.gradle

@@ -60,7 +60,12 @@ dependencyResolutionManagement {
             library('commons-io', 'commons-io', 'commons-io').version('2.15.1')
             library('commons-codec', 'commons-codec', 'commons-codec').version('1.16.0')
             library('commons-compress', 'org.apache.commons', 'commons-compress').version('1.24.0')
-            version('hadoop', '3.3.6')
+            version('parquet', '1.14.1')
+            library('parquet-common', 'org.apache.parquet', 'parquet-common').versionRef('parquet')
+            library('parquet-avro', 'org.apache.parquet', 'parquet-avro').versionRef('parquet')
+            library('parquet-column', 'org.apache.parquet', 'parquet-column').versionRef('parquet')
+            library('parquet-hadoop', 'org.apache.parquet', 'parquet-hadoop').versionRef('parquet')
+            version('hadoop', '3.4.0')
             library('hadoop-common', 'org.apache.hadoop', 'hadoop-common').versionRef('hadoop')
             library('hadoop-mapreduce', 'org.apache.hadoop', 'hadoop-mapreduce-client-core').versionRef('hadoop')
             version('avro', '1.11.3')
@@ -74,7 +79,6 @@ dependencyResolutionManagement {
             version('awaitility', '4.2.0')
             version('spring', '5.3.28')
             version('slf4j', '2.0.6')
-            version('hadoop', '3.3.6')
             library('junit-core', 'org.junit.jupiter', 'junit-jupiter').versionRef('junit')
             library('junit-params', 'org.junit.jupiter', 'junit-jupiter-params').versionRef('junit')
             library('junit-engine', 'org.junit.jupiter', 'junit-jupiter-engine').versionRef('junit')
@@ -88,7 +92,6 @@ dependencyResolutionManagement {
             library('awaitility', 'org.awaitility', 'awaitility').versionRef('awaitility')
             library('spring-test', 'org.springframework', 'spring-test').versionRef('spring')
             library('slf4j-simple', 'org.slf4j', 'slf4j-simple').versionRef('slf4j')
-            library('hadoop-common', 'org.apache.hadoop', 'hadoop-common').versionRef('hadoop')
         }
     }
 }