From 02bcde61f0cf46151a966e102d195366c4be5bc5 Mon Sep 17 00:00:00 2001 From: Arek Burdach Date: Thu, 28 Nov 2024 09:33:46 +0100 Subject: [PATCH] [NU-1897] flink-executor and lite-runtime modules: Added compile-time dependency to http-utils (#7259) * flink-executor and lite-runtime modules: Added compile-time dependency to http-utils * openapi: provided dependency to http utils --- build.sbt | 34 +++++++++++++++++++++++++++------- docs/Changelog.md | 7 +++++-- docs/MigrationGuide.md | 7 +++++-- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/build.sbt b/build.sbt index b9c1fe77c02..e0ad0a586d5 100644 --- a/build.sbt +++ b/build.sbt @@ -811,7 +811,16 @@ lazy val flinkExecutor = (project in flink("executor")) ) }.toList, ) - .dependsOn(flinkComponentsUtils, scenarioCompiler, flinkExtensionsApi, flinkTestUtils % Test) + .dependsOn( + flinkComponentsUtils, + flinkExtensionsApi, + scenarioCompiler, + // Various components uses one of library in stack: sttp -> async-http-client -> netty + // Different versions of netty which is on the bottom of this stack causes NoClassDefFoundError. + // To overcome this problem and reduce size of model jar bundle, we add http utils as a compile time dependency. + httpUtils, + flinkTestUtils % Test + ) lazy val scenarioCompiler = (project in file("scenario-compiler")) .settings(commonSettings) @@ -1355,7 +1364,15 @@ lazy val liteEngineRuntime = (project in lite("runtime")) ) }, ) - .dependsOn(liteComponentsApi, scenarioCompiler, testUtils % Test) + .dependsOn( + liteComponentsApi, + scenarioCompiler, + // Various components uses one of library in stack: sttp -> async-http-client -> netty + // Different versions of netty which is on the bottom of this stack causes NoClassDefFoundError. + // To overcome this problem and reduce size of model jar bundle, we add http utils as a compile time dependency. + httpUtils, + testUtils % Test + ) lazy val liteEngineKafkaIntegrationTest: Project = (project in lite("integration-test")) .configs(IntegrationTest) @@ -1681,6 +1698,9 @@ lazy val processReports = (project in file("designer/processReports")) ) .dependsOn(httpUtils, commonUtils, testUtils % "it,test") +// This dependency is delivered by flink-executor and lite-runtime to ensure the same version of libraries in stack: +// sttp -> async-http-client -> netty. Different versions of netty in model classpath causes NoClassDefFoundError. +// Also, thanks to this approach we reduce size of model jar bundle. lazy val httpUtils = (project in utils("http-utils")) .settings(commonSettings) .settings( @@ -1691,6 +1711,7 @@ lazy val httpUtils = (project in utils("http-utils")) "com.softwaremill.sttp.client3" %% "json-common" % sttpV, "com.softwaremill.sttp.client3" %% "circe" % sttpV, "com.softwaremill.sttp.client3" %% "async-http-client-backend-future" % sttpV, + "io.netty" % "netty-transport-native-epoll" % nettyV, ) } ) @@ -1708,19 +1729,18 @@ lazy val openapiComponents = (project in component("openapi")) .settings( name := "nussknacker-openapi", libraryDependencies ++= Seq( - "io.swagger.core.v3" % "swagger-integration" % swaggerIntegrationV excludeAll ( + "io.swagger.core.v3" % "swagger-integration" % swaggerIntegrationV excludeAll ( ExclusionRule(organization = "jakarta.activation"), ExclusionRule(organization = "jakarta.validation") ), - "io.netty" % "netty-transport-native-epoll" % nettyV, - "org.apache.flink" % "flink-streaming-java" % flinkV % Provided, - "org.scalatest" %% "scalatest" % scalaTestV % "it,test" + "org.apache.flink" % "flink-streaming-java" % flinkV % Provided, + "org.scalatest" %% "scalatest" % scalaTestV % "it,test" ), ) .dependsOn( componentsUtils % Provided, jsonUtils % Provided, - httpUtils, + httpUtils % Provided, requestResponseComponentsUtils % "it,test", flinkComponentsTestkit % "it,test" ) diff --git a/docs/Changelog.md b/docs/Changelog.md index dbf54c66d36..3ba80a5ce2f 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -15,8 +15,11 @@ * [#7123](https://github.com/TouK/nussknacker/pull/7123) Fix deployments for scenarios with dict editors after model reload * [#7162](https://github.com/TouK/nussknacker/pull/7162) Component API enhancement: ability to access information about expression parts used in SpEL template -* [#7257](https://github.com/TouK/nussknacker/pull/7257) components-api module: Replaced wide dependency to `async-http-client-backend-future` - by the narrowest possible dependency to sttp's core +* async-http-client and Netty dependency cleanups + * [#7257](https://github.com/TouK/nussknacker/pull/7257) `components-api` module: Replaced wide dependency to `async-http-client-backend-future` + by the narrowest possible dependency to sttp's core + * [#7259](https://github.com/TouK/nussknacker/pull/7259) `flink-executor` and `lite-runtime` modules: Added compile-time + dependency to `http-utils` (which depends on `async-http-client-backend-future` and indirectly on Netty) ## 1.18 diff --git a/docs/MigrationGuide.md b/docs/MigrationGuide.md index 0b993c910d0..ebda8ade20e 100644 --- a/docs/MigrationGuide.md +++ b/docs/MigrationGuide.md @@ -9,8 +9,11 @@ To see the biggest differences please consult the [changelog](Changelog.md). * [#7116](https://github.com/TouK/nussknacker/pull/7116) Improve missing Flink Kafka Source / Sink TypeInformation * We lost support for old ConsumerRecord constructor supported by Flink 1.14 / 1.15 * If you used Kafka source/sink components in your scenarios then state of these scenarios won't be restored -* [#7257](https://github.com/TouK/nussknacker/pull/7257) `components-api` doesn't depend on `async-http-client-backend-future`. - Instead, it depends on sttp's core. If your component rely on this, you should explicitly add dependency to `async-http-client-backend-future`. +* [#7257](https://github.com/TouK/nussknacker/pull/7257) [#7259](https://github.com/TouK/nussknacker/pull/7259) `components-api` module + doesn't depend on `async-http-client-backend-future`, `http-utils` module is delivered by `flink-executor` and `lite-runtime` modules. + If your component had compile-time dependency to `http-utils`, it should be replaced by provided scope + If your component relied on the fact that `components-api` depends on `async-http-client-backend-future`, + `async-http-client-backend-future` should be added as a provided dependency ## In version 1.18.0