From 5b857c6ba8b79aebe12486b21a980cf669cec6cd Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 28 Oct 2024 13:44:30 -0400 Subject: [PATCH 1/5] Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs (#16284) * WIP on lib toml Signed-off-by: Craig Perkins * SpotlessApply Signed-off-by: Craig Perkins * Remove unnecessary lines Signed-off-by: Craig Perkins * Specify time when dependabot runs Signed-off-by: Craig Perkins * Refer to version from libs.versions.toml Signed-off-by: Craig Perkins * Use version Signed-off-by: Craig Perkins * Specify version catalog Signed-off-by: Craig Perkins * Call .get() Signed-off-by: Craig Perkins * Define version catalog Signed-off-by: Craig Perkins * Use libraries Signed-off-by: Craig Perkins * Downgrade purposefully Signed-off-by: Craig Perkins * Add mavenCentral Signed-off-by: Craig Perkins * Try w/o libraries section Signed-off-by: Craig Perkins * reinstate Signed-off-by: Craig Perkins * Remove version.properties Signed-off-by: Craig Perkins * Update syntax Signed-off-by: Craig Perkins * Change back to weekly Signed-off-by: Craig Perkins * Add grpc Signed-off-by: Craig Perkins * Get relative to project root. Relative path not working on windows bc windows had gradle wrapper path Signed-off-by: Craig Perkins * Add minimal version.properties with only opensearch version to accommodate external references Signed-off-by: Craig Perkins * singularize version.properties Signed-off-by: Craig Perkins * Get rootDir Signed-off-by: Craig Perkins * Fix issue loading snapshot Signed-off-by: Craig Perkins * Limit logic to generating version.properties file within buildSrc Signed-off-by: Craig Perkins * Remove unused exports Signed-off-by: Craig Perkins * Add import Signed-off-by: Craig Perkins * Remove unused code Signed-off-by: Craig Perkins * Remove mavenCentral from publication section Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins * Update reactor-netty version Signed-off-by: Craig Perkins * Only keep versions section in toml Signed-off-by: Craig Perkins * Replaces versions catalog TOML parsing with Gradle's VersionCatalogsExtension Signed-off-by: Andriy Redko * Update bundled_jdk Signed-off-by: Craig Perkins * Update bytebuddy and mockito Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins Signed-off-by: Craig Perkins Signed-off-by: Andriy Redko Signed-off-by: Daniel (dB.) Doubrovkine Co-authored-by: Andriy Redko Co-authored-by: Daniel (dB.) Doubrovkine (cherry picked from commit af7a067e04583d2870af684d70612015e883b63c) --- CHANGELOG.md | 1 + buildSrc/build.gradle | 20 ++++++---- buildSrc/settings.gradle | 8 ++++ buildSrc/version.properties | 76 ----------------------------------- gradle/libs.versions.toml | 80 +++++++++++++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 84 deletions(-) create mode 100644 gradle/libs.versions.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index aeed96fd08a5c..3ae1d8f7e04e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - URI path filtering support in cluster stats API ([#15938](https://github.com/opensearch-project/OpenSearch/pull/15938)) - Add support for renaming aliases during snapshot restore ([#16292](https://github.com/opensearch-project/OpenSearch/pull/16292)) - Add logic in master service to optimize performance and retain detailed logging for critical cluster operations. ([#14795](https://github.com/opensearch-project/OpenSearch/pull/14795)) +- Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284)) ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858)) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 44bbd95e92851..552dd09fc3d9e 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -57,7 +57,7 @@ if (project == rootProject) { // we update the version property to reflect if we are building a snapshot or a release build // we write this back out below to load it in the Build.java which will be shown in rest main action // to indicate this being a snapshot build or a release build. -Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project.file('version.properties')) +Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project) version = props.getProperty("opensearch") def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) { @@ -285,15 +285,19 @@ if (project != rootProject) { } } -// Define this here because we need it early. +// Define this here because we need it early. It uses VersionCatalogsExtension to extract all versions +// and converts to a Java Properties object class VersionPropertiesLoader { - static Properties loadBuildSrcVersion(File input) throws IOException { + static Properties loadBuildSrcVersion(Project project) throws IOException { Properties props = new Properties(); - InputStream is = new FileInputStream(input) - try { - props.load(is) - } finally { - is.close() + + var catalogs = project.extensions.getByType(VersionCatalogsExtension) + var libs = catalogs.named("libs") + libs.getVersionAliases().forEach { + libs.findVersion(it).ifPresent { v -> + // Gradle replaces '_' with '.' so 'google_http_client' becomes 'google.http.client' instead + props.setProperty(it.replaceAll("[.]", "_"), v.requiredVersion) + } } loadBuildSrcVersion(props, System.getProperties()) return props diff --git a/buildSrc/settings.gradle b/buildSrc/settings.gradle index 963177afaa2de..661137fbadae2 100644 --- a/buildSrc/settings.gradle +++ b/buildSrc/settings.gradle @@ -10,3 +10,11 @@ */ include 'reaper' + +dependencyResolutionManagement { + versionCatalogs { + libs { + from(files("../gradle/libs.versions.toml")) + } + } +} diff --git a/buildSrc/version.properties b/buildSrc/version.properties index be0b1807eb4c6..9345c348da16a 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,77 +1 @@ opensearch = 2.19.0 -lucene = 9.12.0 - -bundled_jdk_vendor = adoptium -bundled_jdk = 21.0.5+11 - -# optional dependencies -spatial4j = 0.7 -jts = 1.15.0 -jackson = 2.17.2 -jackson_databind = 2.17.2 -snakeyaml = 2.1 -icu4j = 75.1 -supercsv = 2.4.0 -log4j = 2.21.0 -slf4j = 1.7.36 -asm = 9.7 -jettison = 1.5.4 -woodstox = 6.4.0 -kotlin = 1.7.10 -antlr4 = 4.13.1 -guava = 32.1.1-jre -protobuf = 3.25.5 -jakarta_annotation = 1.3.5 -google_http_client = 1.44.1 -tdigest = 3.2 -hdrhistogram = 2.2.2 -grpc = 1.68.0 - -# when updating the JNA version, also update the version in buildSrc/build.gradle -jna = 5.13.0 - -netty = 4.1.114.Final -joda = 2.12.7 - -# project reactor -reactor_netty = 1.1.23 -reactor = 3.5.20 - -# client dependencies -httpclient = 4.5.14 -httpcore = 4.4.16 -httpasyncclient = 4.1.5 -commonslogging = 1.2 -commonscodec = 1.16.1 -commonslang = 3.14.0 -commonscompress = 1.26.1 -commonsio = 2.16.0 -# plugin dependencies -aws = 2.20.86 -reactivestreams = 1.0.4 - -# when updating this version, you need to ensure compatibility with: -# - plugins/ingest-attachment (transitive dependency, check the upstream POM) -# - distribution/tools/plugin-cli -bouncycastle=1.78 -# test dependencies -randomizedrunner = 2.7.1 -junit = 4.13.2 -hamcrest = 2.1 -mockito = 5.14.1 -objenesis = 3.2 -bytebuddy = 1.15.4 - -# benchmark dependencies -jmh = 1.35 - -# compression -zstd = 1.5.5-5 - -jzlib = 1.1.3 - -resteasy = 6.2.4.Final - -# opentelemetry dependencies -opentelemetry = 1.41.0 -opentelemetrysemconv = 1.27.0-alpha diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 0000000000000..8ff3f6e45397d --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,80 @@ +[versions] +opensearch = "3.0.0" +lucene = "9.12.0" + +bundled_jdk_vendor = "adoptium" +bundled_jdk = "23.0.1+11" + +# optional dependencies +spatial4j = "0.7" +jts = "1.15.0" +jackson = "2.17.2" +jackson_databind = "2.17.2" +snakeyaml = "2.1" +icu4j = "75.1" +supercsv = "2.4.0" +log4j = "2.21.0" +slf4j = "1.7.36" +asm = "9.7" +jettison = "1.5.4" +woodstox = "6.4.0" +kotlin = "1.7.10" +antlr4 = "4.13.1" +guava = "32.1.1-jre" +protobuf = "3.25.5" +jakarta_annotation = "1.3.5" +google_http_client = "1.44.1" +tdigest = "3.3" +hdrhistogram = "2.2.2" +grpc = "1.68.0" + +# when updating the JNA version, also update the version in buildSrc/build.gradle +jna = "5.13.0" + +netty = "4.1.114.Final" +joda = "2.12.7" + +# project reactor +reactor_netty = "1.1.23" +reactor = "3.5.20" + +# client dependencies +httpclient5 = "5.3.1" +httpcore5 = "5.2.5" +httpclient = "4.5.14" +httpcore = "4.4.16" +httpasyncclient = "4.1.5" +commonslogging = "1.2" +commonscodec = "1.16.1" +commonslang = "3.14.0" +commonscompress = "1.26.1" +commonsio = "2.16.0" +# plugin dependencies +aws = "2.20.86" +reactivestreams = "1.0.4" + +# when updating this version, you need to ensure compatibility with: +# - plugins/ingest-attachment (transitive dependency, check the upstream POM) +# - distribution/tools/plugin-cli +bouncycastle="1.78" +# test dependencies +randomizedrunner = "2.7.1" +junit = "4.13.2" +hamcrest = "2.1" +mockito = "5.14.1" +objenesis = "3.2" +bytebuddy = "1.15.4" + +# benchmark dependencies +jmh = "1.35" + +# compression +zstd = "1.5.5-5" + +jzlib = "1.1.3" + +resteasy = "6.2.4.Final" + +# opentelemetry dependencies +opentelemetry = "1.41.0" +opentelemetrysemconv = "1.27.0-alpha" From 6b29babf54b9e20c7144cbdfb216b44fada5cc8c Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 28 Oct 2024 14:15:18 -0400 Subject: [PATCH 2/5] Update os version in libs.versions.toml Signed-off-by: Craig Perkins --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8ff3f6e45397d..b2340e5c82861 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,5 +1,5 @@ [versions] -opensearch = "3.0.0" +opensearch = "2.19.0" lucene = "9.12.0" bundled_jdk_vendor = "adoptium" From adc4728451329c1ae2517748dbd0f16e8de888f9 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 28 Oct 2024 14:26:53 -0400 Subject: [PATCH 3/5] tdigest 3.2 Signed-off-by: Craig Perkins --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b2340e5c82861..34a03dde463fc 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -24,7 +24,7 @@ guava = "32.1.1-jre" protobuf = "3.25.5" jakarta_annotation = "1.3.5" google_http_client = "1.44.1" -tdigest = "3.3" +tdigest = "3.2" hdrhistogram = "2.2.2" grpc = "1.68.0" From 704ea11a779dcfe48da9d107ade50b6b4631a917 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 28 Oct 2024 16:30:08 -0400 Subject: [PATCH 4/5] Update libs.versions.toml Signed-off-by: Andriy Redko --- gradle/libs.versions.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 34a03dde463fc..1eb084944d86e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -39,8 +39,6 @@ reactor_netty = "1.1.23" reactor = "3.5.20" # client dependencies -httpclient5 = "5.3.1" -httpcore5 = "5.2.5" httpclient = "4.5.14" httpcore = "4.4.16" httpasyncclient = "4.1.5" From 59f434b052fadcea493ae44ad2c731cd13b4494b Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 28 Oct 2024 17:38:43 -0400 Subject: [PATCH 5/5] Update libs.versions.toml Signed-off-by: Andriy Redko --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1eb084944d86e..5a55f1c44ed4e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,7 +3,7 @@ opensearch = "2.19.0" lucene = "9.12.0" bundled_jdk_vendor = "adoptium" -bundled_jdk = "23.0.1+11" +bundled_jdk = "21.0.5+11" # optional dependencies spatial4j = "0.7"