diff --git a/.github/dependabot.yml b/.github/dependabot.yml index af83f8b2edac2..e6a1f74299fb5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,7 +11,8 @@ updates: open-pull-requests-limit: 1 package-ecosystem: "github-actions" schedule: - interval: "weekly" + interval: "daily" + time: "21:10" labels: - "dependabot" - "dependencies" diff --git a/build.gradle b/build.gradle index 55b31ca816214..042bad3616fd7 100644 --- a/build.gradle +++ b/build.gradle @@ -38,6 +38,7 @@ import org.apache.tools.ant.taskdefs.condition.Os import org.opensearch.gradle.BuildPlugin import org.opensearch.gradle.Version import org.opensearch.gradle.VersionProperties +import org.opensearch.gradle.VersionPropertiesGenerator import org.opensearch.gradle.info.BuildParams import org.opensearch.gradle.plugin.PluginBuildPlugin import org.opensearch.gradle.tar.SymbolicLinkPreservingTar @@ -119,6 +120,7 @@ subprojects { } } repositories { + mavenCentral() maven { name = 'test' url = "${rootProject.buildDir}/local-test-repo" @@ -733,6 +735,30 @@ tasks.register('checkCompatibility', CheckCompatibilityTask) { description('Checks the compatibility with child components') } +def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) { + destinationFile = file("${buildDir}/version.properties") + comment = 'Generated version properties' + + def libsTomlFile = file("gradle/libs.versions.toml") + def propertiesFile = file("${buildDir}/versions.properties") + + doLast { + def properties = org.opensearch.gradle.VersionPropertiesGenerator.generateVersionProperties(libsTomlFile) + + propertiesFile.withWriter('UTF-8') { writer -> + properties.store(writer, "Generated by Gradle") + } + + println "Generated versions.properties at: ${propertiesFile.absolutePath}" + } +} + +if (project != rootProject) { + processResources { + from(generateVersionProperties) + } +} + allprojects { project -> project.afterEvaluate { if (project.tasks.findByName('publishToMavenLocal')) { diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index aaa1d8bdef85c..3ec10c3646cd1 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -57,19 +57,9 @@ if (project == rootProject) { // we update the version property to reflect if we are building a snapshot or a release build // we write this back out below to load it in the Build.java which will be shown in rest main action // to indicate this being a snapshot build or a release build. -Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project.file('version.properties')) +Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project.file('../gradle/libs.versions.toml')) version = props.getProperty("opensearch") -def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) { - destinationFile = file("${buildDir}/version.properties") - comment = 'Generated version properties' - properties(props) -} - -processResources { - from(generateVersionProperties) -} - /***************************************************************************** * Java version * *****************************************************************************/ @@ -123,6 +113,7 @@ dependencies { api 'org.jruby.joni:joni:2.2.1' api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}" api "org.ajoberstar.grgit:grgit-core:5.2.1" + implementation 'org.tomlj:tomlj:1.1.1' testFixturesApi "junit:junit:${props.getProperty('junit')}" testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}" @@ -297,6 +288,10 @@ class VersionPropertiesLoader { } finally { is.close() } + props.forEach((key, value) -> { + String newValue = value.toString().replace("\"", "") + props.setProperty(key.toString(), newValue) + }); loadBuildSrcVersion(props, System.getProperties()) return props } diff --git a/buildSrc/settings.gradle b/buildSrc/settings.gradle index 963177afaa2de..f220dda2d1aca 100644 --- a/buildSrc/settings.gradle +++ b/buildSrc/settings.gradle @@ -10,3 +10,11 @@ */ include 'reaper' + +//dependencyResolutionManagement { +// versionCatalogs { +// libs { +// from(files("../gradle/libs.versions.toml")) +// } +// } +//} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java b/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java index 4d8b62d95dff1..583f00ba4a766 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java @@ -34,11 +34,15 @@ import org.apache.commons.lang3.StringUtils; import java.io.IOException; -import java.io.InputStream; +import java.nio.file.Path; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import org.tomlj.Toml; +import org.tomlj.TomlParseResult; +import org.tomlj.TomlTable; + /** * Accessor for shared dependency versions used by opensearch, namely the opensearch and lucene versions. */ @@ -123,17 +127,30 @@ public static Map getVersions() { } } - private static Properties getVersionProperties() { - Properties props = new Properties(); - try (InputStream propsStream = VersionProperties.class.getResourceAsStream("/version.properties")) { - if (propsStream == null) { - throw new IllegalStateException("/version.properties resource missing"); + private static void flattenToml(TomlTable tomlTable, Properties properties, String prefix) { + for (String key : tomlTable.keySet()) { + String fullKey = prefix.isEmpty() ? key : prefix + "." + key; + Object value = tomlTable.get(key); + + if (value instanceof TomlTable) { + flattenToml((TomlTable) value, properties, fullKey); + } else { + properties.setProperty(key, value.toString()); } - props.load(propsStream); + } + } + + private static Properties getVersionProperties() { + TomlParseResult toml = null; + try { + toml = Toml.parse(Path.of("gradle/libs.versions.toml")); } catch (IOException e) { - throw new IllegalStateException("Failed to load version properties", e); + throw new RuntimeException(e); } - return props; + + Properties properties = new Properties(); + flattenToml(toml, properties, ""); + return properties; } public static boolean isOpenSearchSnapshot() { diff --git a/buildSrc/src/main/java/org/opensearch/gradle/VersionPropertiesGenerator.java b/buildSrc/src/main/java/org/opensearch/gradle/VersionPropertiesGenerator.java new file mode 100644 index 0000000000000..2e68f4f4a5ac7 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/VersionPropertiesGenerator.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +import java.io.File; +import java.io.FileReader; +import java.util.Properties; + +import org.tomlj.Toml; +import org.tomlj.TomlParseResult; + +/** + * Generator for shared dependency versions used by opensearch, namely the opensearch and lucene versions. + */ +public class VersionPropertiesGenerator { + + public static Properties generateVersionProperties(File tomlFile) throws Exception { + TomlParseResult toml = Toml.parse(new FileReader(tomlFile)); + Properties properties = new Properties(); + + toml.getTable("versions").keySet().forEach(key -> { + String version = toml.getString("versions." + key); + properties.setProperty(key, version); + }); + + return properties; + } +} diff --git a/buildSrc/version.properties b/buildSrc/version.properties deleted file mode 100644 index 4bb6f3af7fbe4..0000000000000 --- a/buildSrc/version.properties +++ /dev/null @@ -1,79 +0,0 @@ -opensearch = 3.0.0 -lucene = 9.12.0 - -bundled_jdk_vendor = adoptium -bundled_jdk = 23+37 - -# optional dependencies -spatial4j = 0.7 -jts = 1.15.0 -jackson = 2.17.2 -jackson_databind = 2.17.2 -snakeyaml = 2.1 -icu4j = 75.1 -supercsv = 2.4.0 -log4j = 2.21.0 -slf4j = 1.7.36 -asm = 9.7 -jettison = 1.5.4 -woodstox = 6.4.0 -kotlin = 1.7.10 -antlr4 = 4.13.1 -guava = 32.1.1-jre -protobuf = 3.25.5 -jakarta_annotation = 1.3.5 -google_http_client = 1.44.1 -tdigest = 3.3 -hdrhistogram = 2.2.2 -grpc = 1.68.0 - -# when updating the JNA version, also update the version in buildSrc/build.gradle -jna = 5.13.0 - -netty = 4.1.114.Final -joda = 2.12.7 - -# project reactor -reactor_netty = 1.1.22 -reactor = 3.5.20 - -# client dependencies -httpclient5 = 5.3.1 -httpcore5 = 5.2.5 -httpclient = 4.5.14 -httpcore = 4.4.16 -httpasyncclient = 4.1.5 -commonslogging = 1.2 -commonscodec = 1.16.1 -commonslang = 3.14.0 -commonscompress = 1.26.1 -commonsio = 2.16.0 -# plugin dependencies -aws = 2.20.86 -reactivestreams = 1.0.4 - -# when updating this version, you need to ensure compatibility with: -# - plugins/ingest-attachment (transitive dependency, check the upstream POM) -# - distribution/tools/plugin-cli -bouncycastle=1.78 -# test dependencies -randomizedrunner = 2.7.1 -junit = 4.13.2 -hamcrest = 2.1 -mockito = 5.14.1 -objenesis = 3.2 -bytebuddy = 1.15.4 - -# benchmark dependencies -jmh = 1.35 - -# compression -zstd = 1.5.5-5 - -jzlib = 1.1.3 - -resteasy = 6.2.4.Final - -# opentelemetry dependencies -opentelemetry = 1.41.0 -opentelemetrysemconv = 1.27.0-alpha diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 0000000000000..52d7eff1512b9 --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,82 @@ +[versions] +opensearch = "3.0.0" +lucene = "9.12.0" + +bundled_jdk_vendor = "adoptium" +bundled_jdk = "21.0.4+7" + +# optional dependencies +spatial4j = "0.7" +jts = "1.15.0" +jackson = "2.17.2" +jackson_databind = "2.17.2" +snakeyaml = "2.1" +icu4j = "75.1" +supercsv = "2.4.0" +log4j = "2.21.0" +slf4j = "1.7.36" +asm = "9.7" +jettison = "1.5.4" +woodstox = "6.4.0" +kotlin = "1.7.10" +antlr4 = "4.13.1" +guava = "32.1.1-jre" +protobuf = "3.25.5" +jakarta_annotation = "1.3.5" +google_http_client = "1.44.1" +tdigest = "3.3" +hdrhistogram = "2.2.2" + +# when updating the JNA version, also update the version in buildSrc/build.gradle +jna = "5.13.0" + +netty = "4.1.114.Final" +joda = "2.12.7" + +# project reactor +reactor_netty = "1.1.22" +reactor = "3.5.20" + +# client dependencies +httpclient5 = "5.3.1" +httpcore5 = "5.2.5" +httpclient = "4.5.14" +httpcore = "4.4.16" +httpasyncclient = "4.1.5" +commonslogging = "1.2" +commonscodec = "1.16.1" +commonslang = "3.14.0" +commonscompress = "1.26.1" +commonsio = "2.16.0" +# plugin dependencies +aws = "2.20.86" +reactivestreams = "1.0.4" + +# when updating this version, you need to ensure compatibility with: +# - plugins/ingest-attachment (transitive dependency, check the upstream POM) +# - distribution/tools/plugin-cli +bouncycastle="1.78" +# test dependencies +randomizedrunner = "2.7.1" +junit = "4.13.2" +hamcrest = "2.1" +mockito = "5.12.0" +objenesis = "3.2" +bytebuddy = "1.14.9" + +# benchmark dependencies +jmh = "1.35" + +# compression +zstd = "1.5.5-5" + +jzlib = "1.1.3" + +resteasy = "6.2.4.Final" + +# opentelemetry dependencies +opentelemetry = "1.41.0" +opentelemetrysemconv = "1.27.0-alpha" + +[libraries] +jodatime = { group = "joda-time", name = "joda-time", version.ref = "joda" } diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 22aa151c92003..774ac3935598c 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -79,7 +79,7 @@ dependencies { api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" - api "joda-time:joda-time:${versions.joda}" + api "joda-time:joda-time:${libs.jodatime}" api "org.slf4j:slf4j-api:${versions.slf4j}" runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" diff --git a/server/build.gradle b/server/build.gradle index eccaf8a127647..efab2ada4a8a4 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -94,7 +94,7 @@ dependencies { api project(":libs:opensearch-cli") // time handling, remove with java 8 time - api "joda-time:joda-time:${versions.joda}" + api(libs.jodatime) // percentiles aggregation api "com.tdunning:t-digest:${versions.tdigest}"