Validating runtime dependencies for conflicts

Makefile

@@ -34,13 +34,13 @@ build/distributions/tiered-storage-for-apache-kafka-$(VERSION).tgz:
 	./gradlew build distTar -x test -x integrationTest -x e2e:test
 
 storage/s3/build/distributions/s3-$(VERSION).tgz:
-	./gradlew build :storage:s3:distTar -x test -x integrationTest -x e2e:test
+	./gradlew build :storage:plugins:s3:distTar -x test -x integrationTest -x e2e:test
 
 storage/gcs/build/distributions/gcs-$(VERSION).tgz:
-	./gradlew build :storage:gcs:distTar -x test -x integrationTest -x e2e:test
+	./gradlew build :storage:plugins:gcs:distTar -x test -x integrationTest -x e2e:test
 
 storage/gcs/build/distributions/azure-$(VERSION).tgz:
-	./gradlew build :storage:azure:distTar -x test -x integrationTest -x e2e:test
+	./gradlew build :storage:plugins:azure:distTar -x test -x integrationTest -x e2e:test
 
 test: build
 	./gradlew test -x e2e:test

build.gradle

@@ -74,6 +74,11 @@ subprojects {
         withJavadocJar()
         withSourcesJar()
     }
+    configurations.runtimeClasspath {
+        resolutionStrategy {
+            failOnVersionConflict()
+        }
+    }
 
     checkstyle {
         toolVersion = "10.12.0"
@@ -123,29 +128,32 @@ subprojects {
 
         azureSdkVersion = "1.2.21"
 
-        testcontainersVersion = "1.19.4"
+        testcontainersVersion = "1.19.7"
 
         testcontainersFakeGcsServerVersion = "0.2.0"
     }
 
     dependencies {
         compileOnly "org.apache.kafka:kafka-clients:$kafkaVersion"
-        compileOnly "org.apache.kafka:kafka-storage-api:$kafkaVersion"
+        compileOnly("org.apache.kafka:kafka-storage-api:$kafkaVersion")
 
+        implementation enforcedPlatform("com.fasterxml.jackson:jackson-bom:$jacksonVersion")
         compileOnly "org.slf4j:slf4j-api:$slf4jVersion"
 
         testImplementation "org.apache.kafka:kafka-clients:$kafkaVersion"
-        testImplementation "org.apache.kafka:kafka-storage-api:$kafkaVersion"
+        testImplementation("org.apache.kafka:kafka-storage-api:$kafkaVersion")
 
-        testImplementation "org.junit.jupiter:junit-jupiter-api:$junitVersion"
-        testImplementation "org.junit.jupiter:junit-jupiter-params:$junitVersion"
-        testImplementation 'org.junit.platform:junit-platform-launcher:$junitPlatformVersion'
-        testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junitVersion"
+        testImplementation(platform("org.junit:junit-bom:$junitVersion"))
+        testImplementation "org.junit.jupiter:junit-jupiter-api"
+        testImplementation "org.junit.jupiter:junit-jupiter-params"
+        testImplementation "org.junit.platform:junit-platform-launcher"
+        testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine"
 
         testImplementation "org.assertj:assertj-core:$assertJVersion"
 
-        testImplementation "org.mockito:mockito-core:$mockitoVersion"
-        testImplementation "org.mockito:mockito-junit-jupiter:$mockitoVersion"
+        testImplementation(platform("org.mockito:mockito-bom:$mockitoVersion"))
+        testImplementation "org.mockito:mockito-core"
+        testImplementation "org.mockito:mockito-junit-jupiter"
 
         testImplementation "org.awaitility:awaitility:$awaitilityVersion"
 
@@ -241,8 +249,8 @@ distributions {
             from(project(":core").configurations.runtimeClasspath)
             from(project(":storage:core").jar)
             from(project(":storage:core").configurations.runtimeClasspath)
-            from(project(":storage:filesystem").jar)
-            from(project(":storage:filesystem").configurations.runtimeClasspath)
+            from(project(":storage:plugins:filesystem").jar)
+            from(project(":storage:plugins:filesystem").configurations.runtimeClasspath)
         }
     }
 }

core/build.gradle

@@ -36,8 +36,8 @@ dependencies {
 
     implementation "org.bouncycastle:bcprov-jdk18on:$bouncyCastleVersion"
 
-    implementation "com.fasterxml.jackson.core:jackson-databind:$jacksonVersion"
-    implementation "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$jacksonVersion"
+    implementation "com.fasterxml.jackson.core:jackson-databind"
+    implementation "com.fasterxml.jackson.datatype:jackson-datatype-jdk8"
 
     implementation "com.github.luben:zstd-jni:$zstdVersion"
 
@@ -47,7 +47,7 @@ dependencies {
     implementation project(':commons')
 
     testImplementation(testFixtures(project(":storage:core")))
-    testImplementation(project(":storage:filesystem"))
+    testImplementation(project(":storage:plugins:filesystem"))
 
     testImplementation "com.github.luben:zstd-jni:$zstdVersion"
     integrationTestImplementation sourceSets.test.output

docker/Dockerfile

@@ -33,17 +33,17 @@ RUN cd /tiered-storage-for-apache-kafka/core \
     && tar -xf tiered-storage-for-apache-kafka-${_VERSION}.tgz --strip-components=1 \
     && rm tiered-storage-for-apache-kafka-${_VERSION}.tgz
 
-COPY storage/s3/build/distributions/s3-${_VERSION}.tgz /tiered-storage-for-apache-kafka/s3
+COPY storage/plugins/s3/build/distributions/s3-${_VERSION}.tgz /tiered-storage-for-apache-kafka/s3
 RUN cd /tiered-storage-for-apache-kafka/s3 \
     && tar -xf s3-${_VERSION}.tgz --strip-components=1 \
     && rm s3-${_VERSION}.tgz
 
-COPY storage/gcs/build/distributions/gcs-${_VERSION}.tgz /tiered-storage-for-apache-kafka/gcs
+COPY storage/plugins/gcs/build/distributions/gcs-${_VERSION}.tgz /tiered-storage-for-apache-kafka/gcs
 RUN cd /tiered-storage-for-apache-kafka/gcs \
     && tar -xf gcs-${_VERSION}.tgz --strip-components=1 \
     && rm gcs-${_VERSION}.tgz
 
-COPY storage/azure/build/distributions/azure-${_VERSION}.tgz /tiered-storage-for-apache-kafka/azure
+COPY storage/plugins/azure/build/distributions/azure-${_VERSION}.tgz /tiered-storage-for-apache-kafka/azure
 RUN cd /tiered-storage-for-apache-kafka/azure \
     && tar -xf azure-${_VERSION}.tgz --strip-components=1 \
     && rm azure-${_VERSION}.tgz

settings.gradle

@@ -18,9 +18,10 @@ rootProject.name = 'tiered-storage-for-apache-kafka'
 include 'core'
 include 'storage'
 include 'storage:core'
-include 'storage:filesystem'
-include 'storage:azure'
-include 'storage:gcs'
-include 'storage:s3'
+include 'storage:plugins'
+include 'storage:plugins:filesystem'
+include 'storage:plugins:azure'
+include 'storage:plugins:gcs'
+include 'storage:plugins:s3'
 include 'e2e'
 include 'commons'

storage/azure/build.gradle → storage/plugins/azure/build.gradle

@@ -17,19 +17,17 @@
 archivesBaseName = "storage-azure"
 
 dependencies {
-    implementation project(":storage:core")
-
     implementation platform("com.azure:azure-sdk-bom:$azureSdkVersion")
-    implementation ("com.azure:azure-identity") {
+    implementation("com.azure:azure-identity") {
         exclude group: "org.slf4j"
+        exclude group: "net.java.dev.jna"
+        exclude group: "com.microsoft.azure", module: "msal4j"
     }
-    implementation ("com.azure:azure-storage-blob") {
+    implementation("com.azure:azure-storage-blob") {
         exclude group: "org.slf4j"
     }
 
     implementation project(":commons")
 
-    testImplementation(testFixtures(project(":storage:core")))
-
-    testImplementation "org.testcontainers:junit-jupiter:$testcontainersVersion"
+    testImplementation("org.testcontainers:junit-jupiter:$testcontainersVersion")
 }

storage/plugins/build.gradle

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2024 Aiven Oy
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+subprojects {
+    configurations {
+        providedRuntime
+        implementation.extendsFrom(providedRuntime)
+    }
+
+    dependencies {
+        implementation project(":storage:core")
+
+        providedRuntime project(":core")
+
+        testImplementation(testFixtures(project(":storage:core")))
+    }
+}
+

storage/filesystem/build.gradle → storage/plugins/filesystem/build.gradle

@@ -17,9 +17,5 @@
 archivesBaseName = "storage-filesystem"
 
 dependencies {
-    implementation project(":storage:core")
-
     implementation "commons-io:commons-io:$apacheCommonsIOVersion"
-
-    testImplementation(testFixtures(project(":storage:core")))
 }

storage/gcs/build.gradle → storage/plugins/gcs/build.gradle

@@ -17,14 +17,14 @@
 archivesBaseName = "storage-gcs"
 
 dependencies {
-    implementation project(":storage:core")
+    implementation("com.google.cloud:google-cloud-storage:$gcpSdkVersion") {
+        exclude group: "com.google.errorprone"
+        exclude group: "org.checkerframework"
+    }
 
-    implementation "com.google.cloud:google-cloud-storage:$gcpSdkVersion"
     implementation project(":commons")
 
-    testImplementation(testFixtures(project(":storage:core")))
+    testImplementation("org.testcontainers:junit-jupiter:$testcontainersVersion")
 
-    testImplementation "org.testcontainers:junit-jupiter:$testcontainersVersion"
-
-    testImplementation "io.aiven:testcontainers-fake-gcs-server:$testcontainersFakeGcsServerVersion"
+    testImplementation("io.aiven:testcontainers-fake-gcs-server:$testcontainersFakeGcsServerVersion")
 }

storage/s3/build.gradle → storage/plugins/s3/build.gradle

@@ -21,20 +21,12 @@ ext {
 }
 
 dependencies {
-    implementation project(":storage:core")
-
-    // Take out jackson libraries out to avoid collision with core jackson dependencies
-    implementation ("software.amazon.awssdk:s3:$awsSdkVersion") {
-        exclude group: "com.fasterxml.jackson.core"
-        exclude group: "com.fasterxml.jackson.dataformat"
+    implementation("software.amazon.awssdk:s3:$awsSdkVersion") {
         exclude group: "org.slf4j"
+        exclude group: "commons-codec"
     }
     // replacing with version used by core
-    implementation "com.fasterxml.jackson.core:jackson-databind:$jacksonVersion"
-    implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:$jacksonVersion"
-    implementation project(':commons')
-
-    testImplementation(testFixtures(project(":storage:core")))
+    implementation project(":commons")
 
     testImplementation "org.testcontainers:junit-jupiter:$testcontainersVersion"
     testImplementation "org.testcontainers:localstack:$testcontainersVersion"