From 6ab46bd85f82d4a17a2b79a40f4b5de0507fe42d Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 3 Aug 2023 12:32:54 -0400 Subject: [PATCH] [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE (#8195) (#8978) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 5 +- .../opensearch/gradle/JavaPackageType.java | 15 ++ .../org/opensearch/gradle/JavaVariant.java | 197 ++++++++++++++++++ .../main/java/org/opensearch/gradle/Jdk.java | 183 +--------------- .../opensearch/gradle/JdkDownloadPlugin.java | 10 +- .../main/java/org/opensearch/gradle/Jre.java | 18 ++ .../opensearch/gradle/JreDownloadPlugin.java | 147 +++++++++++++ .../gradle/OpenSearchDistribution.java | 12 +- .../opensearch/gradle/VersionProperties.java | 44 +++- .../InternalDistributionDownloadPlugin.java | 7 +- .../gradle/test/DistroTestPlugin.java | 21 +- .../opensearch.jre-download.properties | 12 ++ .../DistributionDownloadPluginTests.java | 64 ++++-- distribution/archives/build.gradle | 55 +++-- distribution/build.gradle | 63 +++++- distribution/src/bin/opensearch-env | 9 +- gradle/runtime-jdk-provision.gradle | 2 +- .../org/opensearch/monitor/jvm/JvmInfo.java | 9 +- .../main/java/org/opensearch/node/Node.java | 2 +- settings.gradle | 1 + 21 files changed, 612 insertions(+), 265 deletions(-) create mode 100644 buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java create mode 100644 buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java create mode 100644 buildSrc/src/main/java/org/opensearch/gradle/Jre.java create mode 100644 buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java create mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties diff --git a/CHANGELOG.md b/CHANGELOG.md index 972b20d9ce89e..5eee01239d4bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Introduce new static cluster setting to control slice computation for concurrent segment search. ([#8847](https://github.com/opensearch-project/OpenSearch/pull/8884)) - Add configuration for file cache size to max remote data ratio to prevent oversubscription of file cache ([#8606](https://github.com/opensearch-project/OpenSearch/pull/8606)) - Disallow compression level to be set for default and best_compression index codecs ([#8737]()https://github.com/opensearch-project/OpenSearch/pull/8737) +- [distribution/archives] [Linux] [x64] Provide the variant of the distributions bundled with JRE ([#8195]()https://github.com/opensearch-project/OpenSearch/pull/8195) ### Dependencies - Bump `org.apache.logging.log4j:log4j-core` from 2.17.1 to 2.20.0 ([#8307](https://github.com/opensearch-project/OpenSearch/pull/8307)) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 25fca31587cbc..09c8321e6edb2 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -264,7 +264,10 @@ This repository is split into many top level directories. The most important one ### `distribution` -Builds our tar and zip archives and our rpm and deb packages. +Builds our tar and zip archives and our rpm and deb packages. There are several flavors of the distributions, with the classifier included in the name of the final deliverable (archive or package): + - default (no classifier), the distribution with bundled JDK + - `-no-jdk-` - the distribution without bundled JDK/JRE, assumes the JDK/JRE is going to be pre-installed on the target systems + - `-jre-` - the distribution bundled with JRE (smaller footprint), supported as experimental feature for some platforms ### `libs` diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java b/buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java new file mode 100644 index 0000000000000..2acc335d80df0 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/JavaPackageType.java @@ -0,0 +1,15 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +public enum JavaPackageType { + NONE, + JRE, + JDK +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java b/buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java new file mode 100644 index 0000000000000..5f576984627a8 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/JavaVariant.java @@ -0,0 +1,197 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +import org.gradle.api.Buildable; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.TaskDependency; + +import java.io.File; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +abstract class JavaVariant implements Buildable, Iterable { + + private static final List ALLOWED_ARCHITECTURES = Collections.unmodifiableList( + Arrays.asList("aarch64", "x64", "s390x", "ppc64le") + ); + private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk")); + private static final List ALLOWED_PLATFORMS = Collections.unmodifiableList( + Arrays.asList("darwin", "freebsd", "linux", "mac", "windows") + ); + private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"); + private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)(?:\\+|\\-)(b\\d+?)(@([a-f0-9]{32}))?"); + + private final String name; + private final Configuration configuration; + + private final Property vendor; + private final Property version; + private final Property platform; + private final Property architecture; + private String baseVersion; + private String major; + private String build; + private String hash; + + JavaVariant(String name, Configuration configuration, ObjectFactory objectFactory) { + this.name = name; + this.configuration = configuration; + this.vendor = objectFactory.property(String.class); + this.version = objectFactory.property(String.class); + this.platform = objectFactory.property(String.class); + this.architecture = objectFactory.property(String.class); + } + + public String getName() { + return name; + } + + public String getVendor() { + return vendor.get(); + } + + public void setVendor(final String vendor) { + if (ALLOWED_VENDORS.contains(vendor) == false) { + throw new IllegalArgumentException("unknown vendor [" + vendor + "] for jdk [" + name + "], must be one of " + ALLOWED_VENDORS); + } + this.vendor.set(vendor); + } + + public String getVersion() { + return version.get(); + } + + public void setVersion(String version) { + if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) { + throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]"); + } + parseVersion(version); + this.version.set(version); + } + + public String getPlatform() { + return platform.get(); + } + + public void setPlatform(String platform) { + if (ALLOWED_PLATFORMS.contains(platform) == false) { + throw new IllegalArgumentException( + "unknown platform [" + platform + "] for jdk [" + name + "], must be one of " + ALLOWED_PLATFORMS + ); + } + this.platform.set(platform); + } + + public String getArchitecture() { + return architecture.get(); + } + + public void setArchitecture(final String architecture) { + String jdkArchitecture = translateJdkArchitecture(architecture); + if (ALLOWED_ARCHITECTURES.contains(jdkArchitecture) == false) { + throw new IllegalArgumentException( + "unknown architecture [" + jdkArchitecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES + ); + } + this.architecture.set(jdkArchitecture); + } + + public String getBaseVersion() { + return baseVersion; + } + + public String getMajor() { + return major; + } + + public String getBuild() { + return build; + } + + public String getHash() { + return hash; + } + + public String getPath() { + return configuration.getSingleFile().toString(); + } + + public String getConfigurationName() { + return configuration.getName(); + } + + @Override + public String toString() { + return getPath(); + } + + @Override + public TaskDependency getBuildDependencies() { + return configuration.getBuildDependencies(); + } + + // internal, make this jdks configuration unmodifiable + void finalizeValues() { + if (version.isPresent() == false) { + throw new IllegalArgumentException("version not specified for jdk [" + name + "]"); + } + if (platform.isPresent() == false) { + throw new IllegalArgumentException("platform not specified for jdk [" + name + "]"); + } + if (vendor.isPresent() == false) { + throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]"); + } + if (architecture.isPresent() == false) { + throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]"); + } + version.finalizeValue(); + platform.finalizeValue(); + vendor.finalizeValue(); + architecture.finalizeValue(); + } + + @Override + public Iterator iterator() { + return configuration.iterator(); + } + + private void parseVersion(String version) { + // decompose the bundled jdk version, broken into elements as: [feature, interim, update, build] + // Note the "patch" version is not yet handled here, as it has not yet been used by java. + Matcher jdkVersionMatcher = VERSION_PATTERN.matcher(version); + if (jdkVersionMatcher.matches() == false) { + // Try again with the pre-Java9 version format + jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version); + + if (jdkVersionMatcher.matches() == false) { + throw new IllegalArgumentException("Malformed jdk version [" + version + "]"); + } + } + + baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : ""); + major = jdkVersionMatcher.group(1); + build = jdkVersionMatcher.group(3); + hash = jdkVersionMatcher.group(5); + } + + private String translateJdkArchitecture(String architecture) { + /* + * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands. + */ + return "arm64".equals(architecture) ? "aarch64" : architecture; + } + +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java index 06e857744be2d..3218abe726639 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/Jdk.java @@ -32,140 +32,13 @@ package org.opensearch.gradle; -import org.gradle.api.Buildable; import org.gradle.api.artifacts.Configuration; import org.gradle.api.model.ObjectFactory; -import org.gradle.api.provider.Property; -import org.gradle.api.tasks.TaskDependency; import org.gradle.internal.os.OperatingSystem; -import java.io.File; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class Jdk implements Buildable, Iterable { - - private static final List ALLOWED_ARCHITECTURES = Collections.unmodifiableList( - Arrays.asList("aarch64", "x64", "s390x", "ppc64le") - ); - private static final List ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptium", "adoptopenjdk", "openjdk")); - private static final List ALLOWED_PLATFORMS = Collections.unmodifiableList( - Arrays.asList("darwin", "freebsd", "linux", "mac", "windows") - ); - private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"); - private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)(?:\\+|\\-)(b\\d+?)(@([a-f0-9]{32}))?"); - - private final String name; - private final Configuration configuration; - - private final Property vendor; - private final Property version; - private final Property platform; - private final Property architecture; - private String baseVersion; - private String major; - private String build; - private String hash; - +public class Jdk extends JavaVariant { Jdk(String name, Configuration configuration, ObjectFactory objectFactory) { - this.name = name; - this.configuration = configuration; - this.vendor = objectFactory.property(String.class); - this.version = objectFactory.property(String.class); - this.platform = objectFactory.property(String.class); - this.architecture = objectFactory.property(String.class); - } - - public String getName() { - return name; - } - - public String getVendor() { - return vendor.get(); - } - - public void setVendor(final String vendor) { - if (ALLOWED_VENDORS.contains(vendor) == false) { - throw new IllegalArgumentException("unknown vendor [" + vendor + "] for jdk [" + name + "], must be one of " + ALLOWED_VENDORS); - } - this.vendor.set(vendor); - } - - public String getVersion() { - return version.get(); - } - - public void setVersion(String version) { - if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) { - throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]"); - } - parseVersion(version); - this.version.set(version); - } - - public String getPlatform() { - return platform.get(); - } - - public void setPlatform(String platform) { - if (ALLOWED_PLATFORMS.contains(platform) == false) { - throw new IllegalArgumentException( - "unknown platform [" + platform + "] for jdk [" + name + "], must be one of " + ALLOWED_PLATFORMS - ); - } - this.platform.set(platform); - } - - public String getArchitecture() { - return architecture.get(); - } - - public void setArchitecture(final String architecture) { - String jdkArchitecture = translateJdkArchitecture(architecture); - if (ALLOWED_ARCHITECTURES.contains(jdkArchitecture) == false) { - throw new IllegalArgumentException( - "unknown architecture [" + jdkArchitecture + "] for jdk [" + name + "], must be one of " + ALLOWED_ARCHITECTURES - ); - } - this.architecture.set(jdkArchitecture); - } - - public String getBaseVersion() { - return baseVersion; - } - - public String getMajor() { - return major; - } - - public String getBuild() { - return build; - } - - public String getHash() { - return hash; - } - - public String getPath() { - return configuration.getSingleFile().toString(); - } - - public String getConfigurationName() { - return configuration.getName(); - } - - @Override - public String toString() { - return getPath(); - } - - @Override - public TaskDependency getBuildDependencies() { - return configuration.getBuildDependencies(); + super(name, configuration, objectFactory); } public Object getBinJavaPath() { @@ -190,56 +63,4 @@ private String getHomeRoot() { boolean isOSX = "mac".equals(getPlatform()) || "darwin".equals(getPlatform()); return getPath() + (isOSX ? "/Contents/Home" : ""); } - - // internal, make this jdks configuration unmodifiable - void finalizeValues() { - if (version.isPresent() == false) { - throw new IllegalArgumentException("version not specified for jdk [" + name + "]"); - } - if (platform.isPresent() == false) { - throw new IllegalArgumentException("platform not specified for jdk [" + name + "]"); - } - if (vendor.isPresent() == false) { - throw new IllegalArgumentException("vendor not specified for jdk [" + name + "]"); - } - if (architecture.isPresent() == false) { - throw new IllegalArgumentException("architecture not specified for jdk [" + name + "]"); - } - version.finalizeValue(); - platform.finalizeValue(); - vendor.finalizeValue(); - architecture.finalizeValue(); - } - - @Override - public Iterator iterator() { - return configuration.iterator(); - } - - private void parseVersion(String version) { - // decompose the bundled jdk version, broken into elements as: [feature, interim, update, build] - // Note the "patch" version is not yet handled here, as it has not yet been used by java. - Matcher jdkVersionMatcher = VERSION_PATTERN.matcher(version); - if (jdkVersionMatcher.matches() == false) { - // Try again with the pre-Java9 version format - jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version); - - if (jdkVersionMatcher.matches() == false) { - throw new IllegalArgumentException("Malformed jdk version [" + version + "]"); - } - } - - baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : ""); - major = jdkVersionMatcher.group(1); - build = jdkVersionMatcher.group(3); - hash = jdkVersionMatcher.group(5); - } - - private String translateJdkArchitecture(String architecture) { - /* - * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands. - */ - return "arm64".equals(architecture) ? "aarch64" : architecture; - } - } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java index f638f1931cf62..c359e3a497049 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java @@ -129,13 +129,9 @@ private void setupRepository(Project project, Jdk jdk) { // To distinguish between those, the GA releases have only major version component (fe 17+32), // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of // version separator '.' should be enough. - artifactPattern = "jdk-" - + jdk.getBaseVersion() - + "+" - + jdk.getBuild() - + "/OpenJDK" - + jdk.getMajor() - + (jdk.getBaseVersion().contains(".") ? "U" : "") + artifactPattern = "jdk-" + jdk.getBaseVersion() + "+" + jdk.getBuild() + "/OpenJDK" + jdk.getMajor() + // JDK-20 does use 'U' suffix all the time, no matter it is update or GA release + + (jdk.getBaseVersion().contains(".") || jdk.getBaseVersion().matches("^2\\d+$") ? "U" : "") + "-jdk_[classifier]_[module]_hotspot_" + jdk.getBaseVersion() + "_" diff --git a/buildSrc/src/main/java/org/opensearch/gradle/Jre.java b/buildSrc/src/main/java/org/opensearch/gradle/Jre.java new file mode 100644 index 0000000000000..473bfc4860b80 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/Jre.java @@ -0,0 +1,18 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.model.ObjectFactory; + +public class Jre extends JavaVariant { + Jre(String name, Configuration configuration, ObjectFactory objectFactory) { + super(name, configuration, objectFactory); + } +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java new file mode 100644 index 0000000000000..5a00f41f07a60 --- /dev/null +++ b/buildSrc/src/main/java/org/opensearch/gradle/JreDownloadPlugin.java @@ -0,0 +1,147 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gradle; + +import org.opensearch.gradle.transform.SymbolicLinkPreservingUntarTransform; +import org.opensearch.gradle.transform.UnzipTransform; +import org.gradle.api.GradleException; +import org.gradle.api.NamedDomainObjectContainer; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.dsl.RepositoryHandler; +import org.gradle.api.artifacts.repositories.IvyArtifactRepository; +import org.gradle.api.artifacts.type.ArtifactTypeDefinition; +import org.gradle.api.attributes.Attribute; +import org.gradle.api.internal.artifacts.ArtifactAttributes; + +public class JreDownloadPlugin implements Plugin { + public static final String VENDOR_ADOPTIUM = "adoptium"; + + private static final String REPO_NAME_PREFIX = "jre_repo_"; + private static final String EXTENSION_NAME = "jres"; + public static final String JRE_TRIMMED_PREFIX = "jdk-?\\d.*-jre"; + + @Override + public void apply(Project project) { + Attribute jreAttribute = Attribute.of("jre", Boolean.class); + project.getDependencies().getAttributesSchema().attribute(jreAttribute); + project.getDependencies().getArtifactTypes().maybeCreate(ArtifactTypeDefinition.ZIP_TYPE); + project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> { + transformSpec.getFrom() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE) + .attribute(jreAttribute, true); + transformSpec.getTo() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE) + .attribute(jreAttribute, true); + transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JRE_TRIMMED_PREFIX)); + }); + + ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz"); + project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> { + transformSpec.getFrom() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName()) + .attribute(jreAttribute, true); + transformSpec.getTo() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE) + .attribute(jreAttribute, true); + transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JRE_TRIMMED_PREFIX)); + }); + + NamedDomainObjectContainer jresContainer = project.container(Jre.class, name -> { + Configuration configuration = project.getConfigurations().create("jre_" + name); + configuration.setCanBeConsumed(false); + configuration.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); + configuration.getAttributes().attribute(jreAttribute, true); + Jre jre = new Jre(name, configuration, project.getObjects()); + configuration.defaultDependencies(dependencies -> { + jre.finalizeValues(); + setupRepository(project, jre); + dependencies.add(project.getDependencies().create(dependencyNotation(jre))); + }); + return jre; + }); + project.getExtensions().add(EXTENSION_NAME, jresContainer); + } + + private void setupRepository(Project project, Jre jre) { + RepositoryHandler repositories = project.getRepositories(); + + /* + * Define the appropriate repository for the given JRE vendor and version + * + * For Oracle/OpenJDK/AdoptOpenJDK we define a repository per-version. + */ + String repoName = REPO_NAME_PREFIX + jre.getVendor() + "_" + jre.getVersion(); + String repoUrl; + String artifactPattern; + + if (jre.getVendor().equals(VENDOR_ADOPTIUM)) { + repoUrl = "https://github.com/adoptium/temurin" + jre.getMajor() + "-binaries/releases/download/"; + + if (jre.getMajor().equals("8")) { + // JDK-8 updates are always suffixed with 'U' (fe OpenJDK8U). + artifactPattern = "jdk" + + jre.getBaseVersion() + + "-" + + jre.getBuild() + + "/OpenJDK" + + jre.getMajor() + + "U" + + "-jre_[classifier]_[module]_hotspot_" + + jre.getBaseVersion() + + jre.getBuild() + + ".[ext]"; + } else { + // JDK updates are suffixed with 'U' (fe OpenJDK17U), whereas GA releases are not (fe OpenJDK17). + // To distinguish between those, the GA releases have only major version component (fe 17+32), + // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of + // version separator '.' should be enough. + artifactPattern = "jdk-" + jre.getBaseVersion() + "+" + jre.getBuild() + "/OpenJDK" + jre.getMajor() + // JDK-20 does use 'U' suffix all the time, no matter it is update or GA release + + (jre.getBaseVersion().contains(".") || jre.getBaseVersion().matches("^2\\d+$") ? "U" : "") + + "-jre_[classifier]_[module]_hotspot_" + + jre.getBaseVersion() + + "_" + + jre.getBuild() + + ".[ext]"; + } + } else { + throw new GradleException("Unknown JDK vendor [" + jre.getVendor() + "]"); + } + + // Define the repository if we haven't already + if (repositories.findByName(repoName) == null) { + repositories.ivy(repo -> { + repo.setName(repoName); + repo.setUrl(repoUrl); + repo.metadataSources(IvyArtifactRepository.MetadataSources::artifact); + repo.patternLayout(layout -> layout.artifact(artifactPattern)); + repo.content(repositoryContentDescriptor -> repositoryContentDescriptor.includeGroup(groupName(jre))); + }); + } + } + + @SuppressWarnings("unchecked") + public static NamedDomainObjectContainer getContainer(Project project) { + return (NamedDomainObjectContainer) project.getExtensions().getByName(EXTENSION_NAME); + } + + private static String dependencyNotation(Jre jre) { + String platformDep = jre.getPlatform().equals("darwin") || jre.getPlatform().equals("mac") ? "mac" : jre.getPlatform(); + String extension = jre.getPlatform().equals("windows") ? "zip" : "tar.gz"; + + return groupName(jre) + ":" + platformDep + ":" + jre.getBaseVersion() + ":" + jre.getArchitecture() + "@" + extension; + } + + private static String groupName(Jre jre) { + return jre.getVendor() + "_" + jre.getMajor() + "_jre"; + } + +} diff --git a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java index 968bd13bd4011..0575c23fee9f6 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchDistribution.java @@ -101,7 +101,7 @@ public boolean shouldExtract() { private final Property version; private final Property type; private final Property platform; - private final Property bundledJdk; + private final Property bundledJdk; private final Property failIfUnavailable; private final Configuration extracted; @@ -120,7 +120,7 @@ public boolean shouldExtract() { this.type = objectFactory.property(Type.class); this.type.convention(Type.ARCHIVE); this.platform = objectFactory.property(Platform.class); - this.bundledJdk = objectFactory.property(Boolean.class); + this.bundledJdk = objectFactory.property(JavaPackageType.class); this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true); this.extracted = extractedConfiguration; } @@ -154,8 +154,8 @@ public void setType(Type type) { this.type.set(type); } - public boolean getBundledJdk() { - return bundledJdk.getOrElse(true); + public JavaPackageType getBundledJdk() { + return bundledJdk.getOrElse(JavaPackageType.JDK); } public boolean isDocker() { @@ -163,7 +163,7 @@ public boolean isDocker() { return type == Type.DOCKER; } - public void setBundledJdk(Boolean bundledJdk) { + public void setBundledJdk(JavaPackageType bundledJdk) { this.bundledJdk.set(bundledJdk); } @@ -266,7 +266,7 @@ void finalizeValues() { } if (bundledJdk.isPresent() == false) { - bundledJdk.set(true); + bundledJdk.set(JavaPackageType.JDK); } version.finalizeValue(); diff --git a/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java b/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java index 2b50e59de04d3..4d8b62d95dff1 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java @@ -31,6 +31,8 @@ package org.opensearch.gradle; +import org.apache.commons.lang3.StringUtils; + import java.io.IOException; import java.io.InputStream; import java.util.HashMap; @@ -54,7 +56,7 @@ public static String getLucene() { return lucene; } - public static String getBundledJdk(final String platform) { + public static String getBundledJdk(final String platform, final String arch) { switch (platform) { case "darwin": // fall trough case "mac": @@ -62,7 +64,7 @@ public static String getBundledJdk(final String platform) { case "freebsd": return bundledJdkFreeBSD; case "linux": - return bundledJdkLinux; + return getBundledJdkLinux(arch); case "windows": return bundledJdkWindows; default: @@ -70,6 +72,14 @@ public static String getBundledJdk(final String platform) { } } + public static String getBundledJdk(final String platform) { + return getBundledJdk(platform, null); + } + + public static String getBundledJre(final String platform, final String arch) { + return getBundledJdk(platform, arch); + } + public static String getBundledJdkVendor() { return bundledJdkVendor; } @@ -84,6 +94,10 @@ public static Map getVersions() { private static final String bundledJdkFreeBSD; private static final String bundledJdkLinux; private static final String bundledJdkWindows; + private static final String bundledJdkLinux_arm64; + private static final String bundledJdkLinux_x64; + private static final String bundledJdkLinux_s390x; + private static final String bundledJdkLinux_ppc64le; private static final String bundledJdkVendor; private static final Map versions = new HashMap(); @@ -98,6 +112,12 @@ public static Map getVersions() { bundledJdkLinux = props.getProperty("bundled_jdk_linux", bundledJdk); bundledJdkWindows = props.getProperty("bundled_jdk_windows", bundledJdk); + // Bundled JDKs per architecture (linux platform) + bundledJdkLinux_arm64 = props.getProperty("bundled_jdk_linux_arm64", bundledJdkLinux); + bundledJdkLinux_x64 = props.getProperty("bundled_jdk_linux_x64", bundledJdkLinux); + bundledJdkLinux_s390x = props.getProperty("bundled_jdk_linux_s390x", bundledJdkLinux); + bundledJdkLinux_ppc64le = props.getProperty("bundled_jdk_linux_ppc64le", bundledJdkLinux); + for (String property : props.stringPropertyNames()) { versions.put(property, props.getProperty(property)); } @@ -119,4 +139,24 @@ private static Properties getVersionProperties() { public static boolean isOpenSearchSnapshot() { return opensearch.endsWith("-SNAPSHOT"); } + + private static String getBundledJdkLinux(String arch) { + if (StringUtils.isBlank(arch)) { + return bundledJdkLinux; + } + + switch (arch) { + case "aarch64": + case "arm64": + return bundledJdkLinux_arm64; + case "x64": + return bundledJdkLinux_x64; + case "s390x": + return bundledJdkLinux_s390x; + case "ppc64le": + return bundledJdkLinux_ppc64le; + default: + throw new IllegalArgumentException("unknown platform [" + arch + "] for 'linux' platform"); + } + } } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java index f4368b1cecc59..6a54612320c6c 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionDownloadPlugin.java @@ -37,6 +37,7 @@ import org.opensearch.gradle.DistributionDependency; import org.opensearch.gradle.DistributionDownloadPlugin; import org.opensearch.gradle.DistributionResolution; +import org.opensearch.gradle.JavaPackageType; import org.opensearch.gradle.OpenSearchDistribution; import org.opensearch.gradle.Version; import org.opensearch.gradle.VersionProperties; @@ -99,7 +100,7 @@ private void registerInternalDistributionResolutions(NamedDomainObjectContainer< resolutions.register("bwc", distributionResolution -> distributionResolution.setResolver((project, distribution) -> { BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.unreleasedInfo(Version.fromString(distribution.getVersion())); if (unreleasedInfo != null) { - if (!distribution.getBundledJdk()) { + if (distribution.getBundledJdk() == JavaPackageType.NONE) { throw new GradleException( "Configuring a snapshot bwc distribution ('" + distribution.getName() @@ -167,8 +168,10 @@ private static String distributionProjectName(OpenSearchDistribution distributio ? "" : "-" + architecture.toString().toLowerCase(); - if (distribution.getBundledJdk() == false) { + if (distribution.getBundledJdk() == JavaPackageType.NONE) { projectName += "no-jdk-"; + } else if (distribution.getBundledJdk() == JavaPackageType.JRE) { + projectName += "jre-"; } switch (distribution.getType()) { case ARCHIVE: diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java index d32172758cfce..a311d89583acb 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java @@ -34,6 +34,7 @@ import org.opensearch.gradle.Architecture; import org.opensearch.gradle.DistributionDownloadPlugin; +import org.opensearch.gradle.JavaPackageType; import org.opensearch.gradle.OpenSearchDistribution; import org.opensearch.gradle.Jdk; import org.opensearch.gradle.JdkDownloadPlugin; @@ -71,6 +72,7 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.function.Supplier; import java.util.stream.Stream; @@ -137,7 +139,7 @@ public void apply(Project project) { } if ((distribution.getType() == OpenSearchDistribution.Type.DEB || distribution.getType() == OpenSearchDistribution.Type.RPM) - && distribution.getBundledJdk()) { + && distribution.getBundledJdk() != JavaPackageType.NONE) { for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) { if (version.before("6.3.0")) { continue; // before opening xpack @@ -379,8 +381,8 @@ private List configureDistributions(Project project) { OpenSearchDistribution.Type.RPM, OpenSearchDistribution.Type.DOCKER )) { - for (boolean bundledJdk : Arrays.asList(true, false)) { - if (bundledJdk == false) { + for (JavaPackageType bundledJdk : Set.of(JavaPackageType.NONE, JavaPackageType.JDK)) { + if (bundledJdk == JavaPackageType.NONE) { // We'll never publish an ARM (arm64) build without a bundled JDK. if (architecture == Architecture.ARM64) { continue; @@ -403,8 +405,8 @@ private List configureDistributions(Project project) { OpenSearchDistribution.Platform.LINUX, OpenSearchDistribution.Platform.WINDOWS )) { - for (boolean bundledJdk : Arrays.asList(true, false)) { - if (bundledJdk == false && architecture != Architecture.X64) { + for (JavaPackageType bundledJdk : Set.of(JavaPackageType.NONE, JavaPackageType.JDK)) { + if (bundledJdk == JavaPackageType.NONE && architecture != Architecture.X64) { // We will never publish distributions for non-x86 (amd64) platforms // without a bundled JDK continue; @@ -432,7 +434,7 @@ private static OpenSearchDistribution createDistro( Architecture architecture, OpenSearchDistribution.Type type, OpenSearchDistribution.Platform platform, - boolean bundledJdk, + JavaPackageType bundledJdk, String version ) { String name = distroId(type, platform, bundledJdk, architecture) + "-" + version; @@ -466,11 +468,12 @@ private static boolean isWindows(Project project) { private static String distroId( OpenSearchDistribution.Type type, OpenSearchDistribution.Platform platform, - boolean bundledJdk, + JavaPackageType bundledJdk, Architecture architecture ) { - return (type == OpenSearchDistribution.Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk ? "" : "-no-jdk") - + (architecture == Architecture.X64 ? "" : "-" + architecture.toString().toLowerCase()); + return (type == OpenSearchDistribution.Type.ARCHIVE ? platform + "-" : "") + type + (bundledJdk != JavaPackageType.NONE + ? (bundledJdk == JavaPackageType.JDK ? "" : "-jre") + : "-no-jdk") + (architecture == Architecture.X64 ? "" : "-" + architecture.toString().toLowerCase()); } private static String destructiveDistroTestTaskName(OpenSearchDistribution distro) { diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties new file mode 100644 index 0000000000000..e9253488ffbeb --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.jre-download.properties @@ -0,0 +1,12 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +# + +implementation-class=org.opensearch.gradle.JreDownloadPlugin diff --git a/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java b/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java index 446c94acc7ad4..f9c28d70261b7 100644 --- a/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java +++ b/buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java @@ -45,6 +45,7 @@ import java.util.Arrays; import java.util.TreeSet; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.core.StringContains.containsString; public class DistributionDownloadPluginTests extends GradleUnitTestCase { @@ -76,7 +77,14 @@ public class DistributionDownloadPluginTests extends GradleUnitTestCase { ); public void testVersionDefault() { - OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", null, Type.ARCHIVE, Platform.LINUX, true); + OpenSearchDistribution distro = checkDistro( + createProject(null, false), + "testdistro", + null, + Type.ARCHIVE, + Platform.LINUX, + JavaPackageType.JDK + ); assertEquals(distro.getVersion(), VersionProperties.getOpenSearch()); } @@ -140,18 +148,32 @@ public void testBadVersionFormat() { "badversion", Type.ARCHIVE, Platform.LINUX, - true, + JavaPackageType.JDK, "Invalid version format: 'badversion'" ); } public void testTypeDefault() { - OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", null, Platform.LINUX, true); + OpenSearchDistribution distro = checkDistro( + createProject(null, false), + "testdistro", + "5.0.0", + null, + Platform.LINUX, + JavaPackageType.JDK + ); assertEquals(distro.getType(), Type.ARCHIVE); } public void testPlatformDefault() { - OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", Type.ARCHIVE, null, true); + OpenSearchDistribution distro = checkDistro( + createProject(null, false), + "testdistro", + "5.0.0", + Type.ARCHIVE, + null, + JavaPackageType.JDK + ); assertEquals(distro.getPlatform(), OpenSearchDistribution.CURRENT_PLATFORM); } @@ -168,8 +190,15 @@ public void testPlatformForIntegTest() { } public void testBundledJdkDefault() { - OpenSearchDistribution distro = checkDistro(createProject(null, false), "testdistro", "5.0.0", Type.ARCHIVE, Platform.LINUX, true); - assertTrue(distro.getBundledJdk()); + OpenSearchDistribution distro = checkDistro( + createProject(null, false), + "testdistro", + "5.0.0", + Type.ARCHIVE, + Platform.LINUX, + JavaPackageType.JDK + ); + assertThat(distro.getBundledJdk(), equalTo(JavaPackageType.JDK)); } public void testBundledJdkForIntegTest() { @@ -179,7 +208,7 @@ public void testBundledJdkForIntegTest() { "5.0.0", Type.INTEG_TEST_ZIP, null, - true, + JavaPackageType.JDK, "bundledJdk cannot be set on opensearch distribution [testdistro]" ); } @@ -195,7 +224,7 @@ public void testLocalCurrentVersionIntegTestZip() { public void testLocalCurrentVersionArchives() { for (Platform platform : Platform.values()) { - for (boolean bundledJdk : new boolean[] { true, false }) { + for (JavaPackageType bundledJdk : JavaPackageType.values()) { for (Architecture architecture : Architecture.values()) { // create a new project in each iteration, so that we know we are resolving the only additional project being created Project project = createProject(BWC_MINOR, true); @@ -221,7 +250,7 @@ public void testLocalCurrentVersionArchives() { public void testLocalCurrentVersionPackages() { for (Type packageType : new Type[] { Type.RPM, Type.DEB }) { - for (boolean bundledJdk : new boolean[] { true, false }) { + for (JavaPackageType bundledJdk : JavaPackageType.values()) { Project project = createProject(BWC_MINOR, true); String projectName = projectName(packageType.toString(), bundledJdk); Project packageProject = ProjectBuilder.builder().withParent(packagesProject).withName(projectName).build(); @@ -236,7 +265,7 @@ public void testLocalCurrentVersionPackages() { public void testLocalBwcArchives() { for (Platform platform : Platform.values()) { // note: no non bundled jdk for bwc - String configName = projectName(platform.toString(), true); + String configName = projectName(platform.toString(), JavaPackageType.JDK); configName += (platform == Platform.WINDOWS ? "-zip" : "-tar"); checkBwc("minor", configName, BWC_MINOR_VERSION, Type.ARCHIVE, platform, BWC_MINOR, true); @@ -249,7 +278,7 @@ public void testLocalBwcArchives() { public void testLocalBwcPackages() { for (Type packageType : new Type[] { Type.RPM, Type.DEB }) { // note: no non bundled jdk for bwc - String configName = projectName(packageType.toString(), true); + String configName = projectName(packageType.toString(), JavaPackageType.JDK); checkBwc("minor", configName, BWC_MINOR_VERSION, packageType, null, BWC_MINOR, true); checkBwc("staged", configName, BWC_STAGED_VERSION, packageType, null, BWC_STAGED, true); @@ -264,7 +293,7 @@ private void assertDistroError( String version, Type type, Platform platform, - Boolean bundledJdk, + JavaPackageType bundledJdk, String message ) { IllegalArgumentException e = expectThrows( @@ -280,7 +309,7 @@ private OpenSearchDistribution createDistro( String version, Type type, Platform platform, - Boolean bundledJdk + JavaPackageType bundledJdk ) { NamedDomainObjectContainer distros = DistributionDownloadPlugin.getContainer(project); return distros.create(name, distro -> { @@ -306,7 +335,7 @@ private OpenSearchDistribution checkDistro( String version, Type type, Platform platform, - Boolean bundledJdk + JavaPackageType bundledJdk ) { OpenSearchDistribution distribution = createDistro(project, name, version, type, platform, bundledJdk); distribution.finalizeValues(); @@ -332,7 +361,8 @@ private void checkBwc( Project archiveProject = ProjectBuilder.builder().withParent(bwcProject).withName(projectName).build(); archiveProject.getConfigurations().create(config); archiveProject.getArtifacts().add(config, new File("doesnotmatter")); - createDistro(project, "distro", version.toString(), type, platform, true); + final OpenSearchDistribution distro = createDistro(project, "distro", version.toString(), type, platform, JavaPackageType.JDK); + distro.setArchitecture(Architecture.current()); checkPlugin(project); } @@ -351,7 +381,7 @@ private Project createProject(BwcVersions bwcVersions, boolean isInternal) { return project; } - private static String projectName(String base, boolean bundledJdk) { - return bundledJdk ? base : ("no-jdk-" + base); + private static String projectName(String base, JavaPackageType bundledJdk) { + return (bundledJdk == JavaPackageType.JDK) ? base : ((bundledJdk == JavaPackageType.NONE) ? ("no-jdk-" + base) : "jre-" + base); } } diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index e7137127053e2..161b8008525b4 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -28,9 +28,11 @@ * under the License. */ +import org.opensearch.gradle.JavaPackageType + apply plugin: 'opensearch.internal-distribution-archive-setup' -CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, boolean jdk) { +CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, JavaPackageType java) { return copySpec { into("opensearch-${version}") { into('lib') { @@ -39,19 +41,23 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla into('config') { dirMode 0750 fileMode 0660 - with configFiles(distributionType, jdk) + with configFiles(distributionType, java) from { dirMode 0750 jvmOptionsDir.getParent() } } into('bin') { - with binFiles(distributionType, jdk) + with binFiles(distributionType, java) } - if (jdk) { + if (java == JavaPackageType.JDK) { into("darwin".equals(platform) ? 'jdk.app' : 'jdk') { with jdkFiles(project, platform, architecture) } + } else if (java == JavaPackageType.JRE) { + into("darwin".equals(platform) ? 'jre.app' : 'jre') { + with jreFiles(project, platform, architecture) + } } into('') { from { @@ -73,7 +79,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla rename { 'LICENSE.txt' } } - with noticeFile(jdk) + with noticeFile(java) into('modules') { with modulesFiles } @@ -84,77 +90,84 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla distribution_archives { integTestZip { content { - archiveFiles(transportModulesFiles, 'zip', null, 'x64', false) + archiveFiles(transportModulesFiles, 'zip', null, 'x64', JavaPackageType.NONE) } } darwinTar { archiveClassifier = 'darwin-x64' content { - archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', true) + archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', JavaPackageType.JDK) } } darwinArm64Tar { archiveClassifier = 'darwin-arm64' content { - archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', true) + archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', JavaPackageType.JDK) } } noJdkDarwinTar { archiveClassifier = 'no-jdk-darwin-x64' content { - archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', false) + archiveFiles(modulesFiles('darwin-x64'), 'tar', 'darwin', 'x64', JavaPackageType.NONE) } } noJdkDarwinArm64Tar { archiveClassifier = 'no-jdk-darwin-arm64' content { - archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', false) + archiveFiles(modulesFiles('darwin-arm64'), 'tar', 'darwin', 'arm64', JavaPackageType.NONE) } } freebsdTar { archiveClassifier = 'freebsd-x64' content { - archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', false) + archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', JavaPackageType.NONE) } } noJdkFreebsdTar { archiveClassifier = 'no-jdk-freebsd-x64' content { - archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', false) + archiveFiles(modulesFiles('freebsd-x64'), 'tar', 'freebsd', 'x64', JavaPackageType.NONE) } } linuxArm64Tar { archiveClassifier = 'linux-arm64' content { - archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', true) + archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', JavaPackageType.JDK) } } noJdkLinuxArm64Tar { archiveClassifier = 'no-jdk-linux-arm64' content { - archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', false) + archiveFiles(modulesFiles('linux-arm64'), 'tar', 'linux', 'arm64', JavaPackageType.NONE) } } linuxTar { archiveClassifier = 'linux-x64' content { - archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', true) + archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', JavaPackageType.JDK) } } noJdkLinuxTar { archiveClassifier = 'no-jdk-linux-x64' content { - archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', false) + archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', JavaPackageType.NONE) + } + } + + jreLinuxTar { + archiveClassifier = 'jre-linux-x64' + content { + archiveFiles(modulesFiles('linux-x64'), 'tar', 'linux', 'x64', JavaPackageType.JRE) } } @@ -163,7 +176,7 @@ distribution_archives { linuxS390xTar { archiveClassifier = 'linux-s390x' content { - archiveFiles(modulesFiles('linux-s390x'), 'tar', 'linux', 's390x', false) + archiveFiles(modulesFiles('linux-s390x'), 'tar', 'linux', 's390x', JavaPackageType.NONE) } } @@ -171,28 +184,28 @@ distribution_archives { linuxPpc64leTar { archiveClassifier = 'linux-ppc64le' content { - archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', true) + archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', JavaPackageType.JDK) } } noJdkLinuxPpc64leTar { archiveClassifier = 'no-jdk-linux-ppc64le' content { - archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', false) + archiveFiles(modulesFiles('linux-ppc64le'), 'tar', 'linux', 'ppc64le', JavaPackageType.NONE) } } windowsZip { archiveClassifier = 'windows-x64' content { - archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', true) + archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', JavaPackageType.JDK) } } noJdkWindowsZip { archiveClassifier = 'no-jdk-windows-x64' content { - archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', false) + archiveFiles(modulesFiles('windows-x64'), 'zip', 'windows', 'x64', JavaPackageType.NONE) } } } diff --git a/distribution/build.gradle b/distribution/build.gradle index 21e1cf3c17875..bbbef40f51c57 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -36,6 +36,7 @@ import org.opensearch.gradle.MavenFilteringHack import org.opensearch.gradle.NoticeTask import org.opensearch.gradle.VersionProperties import org.opensearch.gradle.info.BuildParams +import org.opensearch.gradle.JavaPackageType import java.nio.file.Files import java.nio.file.Path @@ -275,15 +276,16 @@ project(':test:external-modules').subprojects.each { Project testModule -> configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { apply plugin: 'opensearch.jdk-download' + apply plugin: 'opensearch.jre-download' apply plugin: 'opensearch.repositories' // Setup all required JDKs project.jdks { ['darwin', 'linux', 'windows'].each { platform -> (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64', 's390x', 'ppc64le'] : ['x64']).each { architecture -> - "bundled_${platform}_${architecture}" { + "bundled_jdk_${platform}_${architecture}" { it.platform = platform - it.version = VersionProperties.getBundledJdk(platform) + it.version = VersionProperties.getBundledJdk(platform, architecture) it.vendor = VersionProperties.bundledJdkVendor it.architecture = architecture } @@ -291,6 +293,20 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } + // Setup all required JREs + project.jres { + ['darwin', 'linux', 'windows'].each { platform -> + (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64', 's390x', 'ppc64le'] : ['x64']).each { architecture -> + "bundled_jre_${platform}_${architecture}" { + it.platform = platform + it.version = VersionProperties.getBundledJre(platform, architecture) + it.vendor = VersionProperties.bundledJdkVendor + it.architecture = architecture + } + } + } + } + // TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run... /***************************************************************************** * Properties to expand when copying packaging files * @@ -377,20 +393,20 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { from buildTransportModulesTaskProvider } - configFiles = { distributionType, jdk -> + configFiles = { distributionType, java -> copySpec { with copySpec { // main config files, processed with distribution specific substitutions from '../src/config' exclude 'log4j2.properties' // this is handled separately below - MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk)) + MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java)) } from project(':distribution').buildLog4jConfig from project(':distribution').buildConfig } } - binFiles = { distributionType, jdk -> + binFiles = { distributionType, java -> copySpec { // non-windows files, for all distributions with copySpec { @@ -398,7 +414,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { exclude '*.exe' exclude '*.bat' eachFile { it.setMode(0755) } - MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk)) + MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java)) } // windows files, only for zip if (distributionType == 'zip') { @@ -406,7 +422,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { from '../src/bin' include '*.bat' filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf')) - MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk)) + MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, java)) } with copySpec { from '../src/bin' @@ -424,12 +440,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } - noticeFile = { jdk -> + noticeFile = { java -> copySpec { if (project.name == 'integ-test-zip') { from buildServerNoticeTaskProvider } else { - if (jdk) { + if (java != JavaPackageType.NONE) { from buildNoticeTaskProvider } else { from buildNoJdkNoticeTaskProvider @@ -446,7 +462,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { if ("arm64".equals(architecture)) { architecture = "aarch64" } - from project.jdks."bundled_${platform}_${architecture}" + from project.jdks."bundled_jdk_${platform}_${architecture}" exclude "demo/**" /* * The Contents/MacOS directory interferes with notarization, and is unused by our distribution, so we exclude @@ -465,6 +481,31 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { } } } + + jreFiles = { Project project, String platform, String architecture -> + return copySpec { + /* + * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands. + */ + if ("arm64".equals(architecture)) { + architecture = "aarch64" + } + from project.jres."bundled_jre_${platform}_${architecture}" + exclude "demo/**" + /* + * The Contents/MacOS directory interferes with notarization, and is unused by our distribution, so we exclude + * it from the build. + */ + if ("darwin".equals(platform)) { + exclude "Contents/MacOS" + } + eachFile { FileCopyDetails details -> + if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') { + details.mode = 0755 + } + } + } + } } } @@ -583,7 +624,7 @@ subprojects { ], 'opensearch.bundled_jdk': [ - 'def': jdk ? 'true' : 'false' + 'def': jdk != JavaPackageType.NONE ? true : false ], 'license.name': [ diff --git a/distribution/src/bin/opensearch-env b/distribution/src/bin/opensearch-env index f94824576b122..cd7a0b2b4520b 100644 --- a/distribution/src/bin/opensearch-env +++ b/distribution/src/bin/opensearch-env @@ -46,7 +46,7 @@ fi # now set the classpath OPENSEARCH_CLASSPATH="$OPENSEARCH_HOME/lib/*" -# now set the path to java: OPENSEARCH_JAVA_HOME -> JAVA_HOME -> bundled JDK +# now set the path to java: OPENSEARCH_JAVA_HOME -> JAVA_HOME -> bundled JRE -> bundled JDK if [ ! -z "$OPENSEARCH_JAVA_HOME" ]; then JAVA="$OPENSEARCH_JAVA_HOME/bin/java" JAVA_TYPE="OPENSEARCH_JAVA_HOME" @@ -57,13 +57,18 @@ else if [ $OS = "darwin" ]; then # macOS bundled Java JAVA="$OPENSEARCH_HOME/jdk.app/Contents/Home/bin/java" + JAVA_TYPE="bundled jdk" elif [ $OS = "freebsd" ]; then # using FreeBSD default java from ports if JAVA_HOME is not set JAVA="/usr/local/bin/java" + JAVA_TYPE="bundled jdk" + elif [ -d "$OPENSEARCH_HOME/jre" ]; then + JAVA="$OPENSEARCH_HOME/jre/bin/java" + JAVA_TYPE="bundled jre" else JAVA="$OPENSEARCH_HOME/jdk/bin/java" + JAVA_TYPE="bundled jdk" fi - JAVA_TYPE="bundled jdk" fi if [ ! -x "$JAVA" ]; then diff --git a/gradle/runtime-jdk-provision.gradle b/gradle/runtime-jdk-provision.gradle index cb4bb834f89c9..6bff164633c25 100644 --- a/gradle/runtime-jdk-provision.gradle +++ b/gradle/runtime-jdk-provision.gradle @@ -32,7 +32,7 @@ if (BuildParams.getIsRuntimeJavaHomeSet()) { jdks { provisioned_runtime { vendor = VersionProperties.bundledJdkVendor - version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase()) + version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase(), Architecture.current().name().toLowerCase()) platform = OS.current().name().toLowerCase() architecture = Architecture.current().name().toLowerCase() } diff --git a/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java b/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java index dd7006a8bbea0..22466bc14b254 100644 --- a/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java +++ b/server/src/main/java/org/opensearch/monitor/jvm/JvmInfo.java @@ -172,7 +172,7 @@ public class JvmInfo implements ReportingService.Info { } final boolean bundledJdk = Booleans.parseBoolean(System.getProperty("opensearch.bundled_jdk", Boolean.FALSE.toString())); - final Boolean usingBundledJdk = bundledJdk ? usingBundledJdk() : null; + final Boolean usingBundledJdkOrJre = bundledJdk ? usingBundledJdkOrJre() : null; INSTANCE = new JvmInfo( JvmPid.getPid(), @@ -181,7 +181,7 @@ public class JvmInfo implements ReportingService.Info { runtimeMXBean.getVmVersion(), runtimeMXBean.getVmVendor(), bundledJdk, - usingBundledJdk, + usingBundledJdkOrJre, runtimeMXBean.getStartTime(), configuredInitialHeapSize, configuredMaxHeapSize, @@ -202,7 +202,7 @@ public class JvmInfo implements ReportingService.Info { } @SuppressForbidden(reason = "PathUtils#get") - private static boolean usingBundledJdk() { + private static boolean usingBundledJdkOrJre() { /* * We are using the bundled JDK if java.home is the jdk sub-directory of our working directory. This is because we always set * the working directory of Elasticsearch to home, and the bundled JDK is in the jdk sub-directory there. @@ -212,7 +212,8 @@ private static boolean usingBundledJdk() { if (Constants.MAC_OS_X) { return PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jdk.app/Contents/Home").toAbsolutePath()); } else { - return PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jdk").toAbsolutePath()); + return PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jre").toAbsolutePath()) + || PathUtils.get(javaHome).equals(PathUtils.get(userDir).resolve("jdk").toAbsolutePath()); } } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 5a397e19a52a0..0a58aba8c35e6 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -429,7 +429,7 @@ protected Node( Constants.JVM_VERSION ); if (jvmInfo.getBundledJdk()) { - logger.info("JVM home [{}], using bundled JDK [{}]", System.getProperty("java.home"), jvmInfo.getUsingBundledJdk()); + logger.info("JVM home [{}], using bundled JDK/JRE [{}]", System.getProperty("java.home"), jvmInfo.getUsingBundledJdk()); } else { logger.info("JVM home [{}]", System.getProperty("java.home")); deprecationLogger.deprecate( diff --git a/settings.gradle b/settings.gradle index 1e038e7fea434..c04b5997d49b1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -53,6 +53,7 @@ List projects = [ 'distribution:archives:no-jdk-linux-ppc64le-tar', 'distribution:archives:linux-tar', 'distribution:archives:no-jdk-linux-tar', + 'distribution:archives:jre-linux-tar', 'distribution:docker', 'distribution:docker:docker-arm64-build-context', 'distribution:docker:docker-arm64-export',