Skip to content

Commit

Permalink
Merge branch 'lib-toml'
Browse files Browse the repository at this point in the history
  • Loading branch information
cwperks committed Oct 10, 2024
2 parents 96082f7 + 221104d commit 9bc3d4a
Show file tree
Hide file tree
Showing 6 changed files with 178 additions and 20 deletions.
25 changes: 25 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
import org.opensearch.gradle.BuildPlugin
import org.opensearch.gradle.Version
import org.opensearch.gradle.VersionProperties
import org.opensearch.gradle.VersionPropertiesGenerator
import org.opensearch.gradle.info.BuildParams
import org.opensearch.gradle.plugin.PluginBuildPlugin
import org.opensearch.gradle.tar.SymbolicLinkPreservingTar
Expand Down Expand Up @@ -733,6 +734,30 @@ tasks.register('checkCompatibility', CheckCompatibilityTask) {
description('Checks the compatibility with child components')
}

def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) {
destinationFile = file("${buildDir}/version.properties")
comment = 'Generated version properties'

def libsTomlFile = file("gradle/libs.versions.toml")
def propertiesFile = file("${buildDir}/versions.properties")

doLast {
def properties = org.opensearch.gradle.VersionPropertiesGenerator.generateVersionProperties(libsTomlFile)

propertiesFile.withWriter('UTF-8') { writer ->
properties.store(writer, "Generated by Gradle")
}

println "Generated versions.properties at: ${propertiesFile.absolutePath}"
}
}

if (project != rootProject) {
processResources {
from(generateVersionProperties)
}
}

allprojects { project ->
project.afterEvaluate {
if (project.tasks.findByName('publishToMavenLocal')) {
Expand Down
17 changes: 6 additions & 11 deletions buildSrc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -57,19 +57,9 @@ if (project == rootProject) {
// we update the version property to reflect if we are building a snapshot or a release build
// we write this back out below to load it in the Build.java which will be shown in rest main action
// to indicate this being a snapshot build or a release build.
Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project.file('version.properties'))
Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project.file('../gradle/libs.versions.toml'))
version = props.getProperty("opensearch")

def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) {
destinationFile = file("${buildDir}/version.properties")
comment = 'Generated version properties'
properties(props)
}

processResources {
from(generateVersionProperties)
}

/*****************************************************************************
* Java version *
*****************************************************************************/
Expand Down Expand Up @@ -123,6 +113,7 @@ dependencies {
api 'org.jruby.joni:joni:2.2.1'
api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
api "org.ajoberstar.grgit:grgit-core:5.2.1"
implementation 'org.tomlj:tomlj:1.1.1'

testFixturesApi "junit:junit:${props.getProperty('junit')}"
testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
Expand Down Expand Up @@ -297,6 +288,10 @@ class VersionPropertiesLoader {
} finally {
is.close()
}
props.forEach((key, value) -> {
String newValue = value.toString().replace("\"", "")
props.setProperty(key.toString(), newValue)
});
loadBuildSrcVersion(props, System.getProperties())
return props
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,15 @@
import org.apache.commons.lang3.StringUtils;

import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

import org.tomlj.Toml;
import org.tomlj.TomlParseResult;
import org.tomlj.TomlTable;

/**
* Accessor for shared dependency versions used by opensearch, namely the opensearch and lucene versions.
*/
Expand Down Expand Up @@ -123,17 +127,30 @@ public static Map<String, String> getVersions() {
}
}

private static Properties getVersionProperties() {
Properties props = new Properties();
try (InputStream propsStream = VersionProperties.class.getResourceAsStream("/version.properties")) {
if (propsStream == null) {
throw new IllegalStateException("/version.properties resource missing");
private static void flattenToml(TomlTable tomlTable, Properties properties, String prefix) {
for (String key : tomlTable.keySet()) {
String fullKey = prefix.isEmpty() ? key : prefix + "." + key;
Object value = tomlTable.get(key);

if (value instanceof TomlTable) {
flattenToml((TomlTable) value, properties, fullKey);
} else {
properties.setProperty(key, value.toString());
}
props.load(propsStream);
}
}

private static Properties getVersionProperties() {
TomlParseResult toml = null;
try {
toml = Toml.parse(Path.of("gradle/libs.versions.toml"));
} catch (IOException e) {
throw new IllegalStateException("Failed to load version properties", e);
throw new RuntimeException(e);
}
return props;

Properties properties = new Properties();
flattenToml(toml, properties, "");
return properties;
}

public static boolean isOpenSearchSnapshot() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.gradle;

import java.io.File;
import java.io.FileReader;
import java.util.Properties;

import org.tomlj.Toml;
import org.tomlj.TomlParseResult;

/**
* Generator for shared dependency versions used by opensearch, namely the opensearch and lucene versions.
*/
public class VersionPropertiesGenerator {

public static Properties generateVersionProperties(File tomlFile) throws Exception {
TomlParseResult toml = Toml.parse(new FileReader(tomlFile));
Properties properties = new Properties();

toml.getTable("versions").keySet().forEach(key -> {
String version = toml.getString("versions." + key);
properties.setProperty(key, version);
});

return properties;
}
}
79 changes: 79 additions & 0 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
[versions]
opensearch = "3.0.0"
lucene = "9.12.0"

bundled_jdk_vendor = "adoptium"
bundled_jdk = "21.0.4+7"

# optional dependencies
spatial4j = "0.7"
jts = "1.15.0"
jackson = "2.17.2"
jackson_databind = "2.17.2"
snakeyaml = "2.1"
icu4j = "75.1"
supercsv = "2.4.0"
log4j = "2.21.0"
slf4j = "1.7.36"
asm = "9.7"
jettison = "1.5.4"
woodstox = "6.4.0"
kotlin = "1.7.10"
antlr4 = "4.13.1"
guava = "32.1.1-jre"
protobuf = "3.25.5"
jakarta_annotation = "1.3.5"
google_http_client = "1.44.1"
tdigest = "3.3"
hdrhistogram = "2.2.2"

# when updating the JNA version, also update the version in buildSrc/build.gradle
jna = "5.13.0"

netty = "4.1.114.Final"
joda = "2.12.7"

# project reactor
reactor_netty = "1.1.22"
reactor = "3.5.20"

# client dependencies
httpclient5 = "5.3.1"
httpcore5 = "5.2.5"
httpclient = "4.5.14"
httpcore = "4.4.16"
httpasyncclient = "4.1.5"
commonslogging = "1.2"
commonscodec = "1.16.1"
commonslang = "3.14.0"
commonscompress = "1.26.1"
commonsio = "2.16.0"
# plugin dependencies
aws = "2.20.86"
reactivestreams = "1.0.4"

# when updating this version, you need to ensure compatibility with:
# - plugins/ingest-attachment (transitive dependency, check the upstream POM)
# - distribution/tools/plugin-cli
bouncycastle="1.78"
# test dependencies
randomizedrunner = "2.7.1"
junit = "4.13.2"
hamcrest = "2.1"
mockito = "5.12.0"
objenesis = "3.2"
bytebuddy = "1.14.9"

# benchmark dependencies
jmh = "1.35"

# compression
zstd = "1.5.5-5"

jzlib = "1.1.3"

resteasy = "6.2.4.Final"

# opentelemetry dependencies
opentelemetry = "1.41.0"
opentelemetrysemconv = "1.27.0-alpha"
8 changes: 8 additions & 0 deletions settings.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,14 @@ include projects.toArray(new String[0])
project(':build-tools').projectDir = new File(rootProject.projectDir, 'buildSrc')
project(':build-tools:reaper').projectDir = new File(rootProject.projectDir, 'buildSrc/reaper')

//dependencyResolutionManagement {
// versionCatalogs {
// libs {
// from(files("gradle/libs.versions.toml"))
// }
// }
//}

project(":libs").children.each { libsProject ->
libsProject.name = "opensearch-${libsProject.name}"
}
Expand Down

0 comments on commit 9bc3d4a

Please sign in to comment.