Resolve runtime java via Gradle tool chain provider (elastic#95319)

Using gradle toolchain support in gradle requires refactoring how the composite build is composed. 
We added three toolchain resolver
1. Resolver for resolving defined bundled version from oracle as openjdk
2. Resolve all available jdks from Adoption
3. Resolve archived Oracle jdk distributions. 

We should be able to remove the JdkDownloadPlugin altogether without having that in place, but we'll do that in a separate effort.

Fixes elastic#95094

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.gradle.internal.conventions;
 
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
+import org.elasticsearch.gradle.internal.conventions.util.Util;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.model.ObjectFactory;
@@ -35,10 +36,7 @@ class GitInfoPlugin implements Plugin<Project> {
 
     @Override
     public void apply(Project project) {
-        File rootDir = (project.getGradle().getParent() == null) ?
-                project.getRootDir() :
-                project.getGradle().getParent().getRootProject().getRootDir();
-
+        File rootDir = Util.locateElasticsearchWorkspace(project.getGradle());
         gitInfo = objectFactory.property(GitInfo.class).value(factory.provider(() ->
             GitInfo.gitInfo(rootDir)
         ));

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/util/Util.java

@@ -21,6 +21,7 @@
 
 import javax.annotation.Nullable;
 import java.io.File;
+import java.util.Collection;
 import java.util.Optional;
 import java.util.function.Supplier;
 
@@ -105,7 +106,6 @@ private static boolean isJavaExtensionAvailable(Project project) {
         return project.getExtensions().getByType(JavaPluginExtension.class) == null;
     }
 
-
     public static Object toStringable(Supplier<String> getter) {
         return new Object() {
             @Override
@@ -119,21 +119,31 @@ public static SourceSetContainer getJavaSourceSets(Project project) {
         return project.getExtensions().getByType(JavaPluginExtension.class).getSourceSets();
     }
 
-    public static File locateElasticsearchWorkspace(Gradle project) {
-        if (project.getParent() == null) {
-            // See if any of these included builds is the Elasticsearch project
-            for (IncludedBuild includedBuild : project.getIncludedBuilds()) {
-                File versionProperties = new File(includedBuild.getProjectDir(), "build-tools-internal/version.properties");
-                if (versionProperties.exists()) {
-                    return includedBuild.getProjectDir();
+        public static File locateElasticsearchWorkspace(Gradle gradle) {
+            if(gradle.getRootProject().getName().startsWith("build-tools")) {
+                File buildToolsParent = gradle.getRootProject().getRootDir().getParentFile();
+                if(versionFileExists(buildToolsParent)) {
+                    return buildToolsParent;
                 }
+                return buildToolsParent;
             }
+            if (gradle.getParent() == null) {
+                // See if any of these included builds is the Elasticsearch gradle
+                for (IncludedBuild includedBuild : gradle.getIncludedBuilds()) {
+                    if (versionFileExists(includedBuild.getProjectDir())) {
+                        return includedBuild.getProjectDir();
+                    }
+                }
 
-            // Otherwise assume this project is the root elasticsearch workspace
-            return project.getRootProject().getRootDir();
-        } else {
-            // We're an included build, so keep looking
-            return locateElasticsearchWorkspace(project.getParent());
+                // Otherwise assume this gradle is the root elasticsearch workspace
+                return gradle.getRootProject().getRootDir();
+            } else {
+                // We're an included build, so keep looking
+                return locateElasticsearchWorkspace(gradle.getParent());
+            }
         }
+
+    private static boolean versionFileExists(File rootDir) {
+        return new File(rootDir, "build-tools-internal/version.properties").exists();
     }
 }

build-tools-internal/build.gradle

@@ -107,6 +107,10 @@ gradlePlugin {
       id = 'elasticsearch.internal-test-rerun'
       implementationClass = 'org.elasticsearch.gradle.internal.test.rerun.TestRerunPlugin'
     }
+    javaToolChainPlugin {
+      id = 'elasticsearch.java-toolchain'
+      implementationClass = 'org.elasticsearch.gradle.internal.toolchain.JavaToolChainResolverPlugin'
+    }
     javaDoc {
       id = 'elasticsearch.java-doc'
       implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavadocPlugin'

build-tools-internal/settings.gradle

@@ -1,3 +1,8 @@
+pluginManagement {
+    includeBuild "../build-conventions"
+    includeBuild "../build-tools"
+}
+
 dependencyResolutionManagement {
     versionCatalogs {
         buildLibs {

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -245,20 +245,20 @@ Node parseXml(Object path) {
   return xml
 }
 
-Pair<File, IncludedBuild> locateElasticsearchWorkspace(Gradle project) {
-  if (project.parent == null) {
-    // See if any of these included builds is the Elasticsearch project
-    for (IncludedBuild includedBuild : project.includedBuilds) {
+Pair<File, IncludedBuild> locateElasticsearchWorkspace(Gradle gradle) {
+  if (gradle.parent == null) {
+    // See if any of these included builds is the Elasticsearch gradle
+    for (IncludedBuild includedBuild : gradle.includedBuilds) {
       File versionProperties = new File(includedBuild.projectDir, 'build-tools-internal/version.properties')
       if (versionProperties.exists()) {
         return Pair.of(includedBuild.projectDir, includedBuild)
       }
     }
 
-    // Otherwise assume this project is the root elasticsearch workspace
-    return Pair.of(project.getRootProject().getRootDir(), null)
+    // Otherwise assume this gradle is the root elasticsearch workspace
+    return Pair.of(gradle.getRootProject().getRootDir(), null)
   } else {
     // We're an included build, so keep looking
-    return locateElasticsearchWorkspace(project.parent)
+    return locateElasticsearchWorkspace(gradle.parent)
   }
 }

build-tools-internal/src/main/groovy/elasticsearch.runtime-jdk-provision.gradle

@@ -35,8 +35,12 @@ configure(allprojects) {
             test.executable = "${BuildParams.runtimeJavaHome}/bin/java" +
                     (OS.current() == OS.WINDOWS ? '.exe' : '')
         } else {
-            test.dependsOn(project.jdks.provisioned_runtime)
-            test.executable = rootProject.jdks.provisioned_runtime.getBinJavaPath()
+            test.javaLauncher = javaToolchains.launcherFor {
+                languageVersion = JavaLanguageVersion.of(VersionProperties.bundledJdkMajorVersion)
+                vendor = VersionProperties.bundledJdkVendor == "openjdk" ?
+                        JvmVendorSpec.ORACLE :
+                        JvmVendorSpec.matching(VersionProperties.bundledJdkVendor)
+            }
         }
     }
     project.plugins.withId("elasticsearch.testclusters") { testClustersPlugin ->

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java

@@ -22,6 +22,13 @@
 
 import java.util.Arrays;
 
+/**
+ * @deprecated We wanna get rid from this and custom jdk downloads via this plugin and
+ * make leverage the gradle toolchain resolver capabilities.
+ *
+ * @See @org.elasticsearch.gradle.internal.toolchain.JavaToolChainResolverPlugin
+ * */
+@Deprecated
 public class JdkDownloadPlugin implements Plugin<Project> {
 
     public static final String VENDOR_ADOPTIUM = "adoptium";
@@ -161,7 +168,6 @@ public static NamedDomainObjectContainer<Jdk> getContainer(Project project) {
     private static String dependencyNotation(Jdk jdk) {
         String platformDep = isJdkOnMacOsPlatform(jdk) ? (jdk.getVendor().equals(VENDOR_ADOPTIUM) ? "mac" : "macos") : jdk.getPlatform();
         String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
-
         return groupName(jdk) + ":" + platformDep + ":" + jdk.getBaseVersion() + ":" + jdk.getArchitecture() + "@" + extension;
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java

@@ -20,18 +20,24 @@
 import org.gradle.api.Project;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
+import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.plugins.JvmToolchainsPlugin;
+import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.internal.jvm.Jvm;
 import org.gradle.internal.jvm.inspection.JvmInstallationMetadata;
 import org.gradle.internal.jvm.inspection.JvmMetadataDetector;
 import org.gradle.internal.jvm.inspection.JvmVendor;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
+import org.gradle.jvm.toolchain.JavaLauncher;
+import org.gradle.jvm.toolchain.JavaToolchainService;
 import org.gradle.jvm.toolchain.JavaToolchainSpec;
 import org.gradle.jvm.toolchain.JvmVendorSpec;
 import org.gradle.jvm.toolchain.internal.InstallationLocation;
 import org.gradle.jvm.toolchain.internal.JavaInstallationRegistry;
 import org.gradle.util.GradleVersion;
+import org.jetbrains.annotations.NotNull;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -56,26 +62,33 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class);
     private static final String DEFAULT_VERSION_JAVA_FILE_PATH = "server/src/main/java/org/elasticsearch/Version.java";
 
+    private ObjectFactory objectFactory;
     private final JavaInstallationRegistry javaInstallationRegistry;
     private final JvmMetadataDetector metadataDetector;
     private final ProviderFactory providers;
+    private JavaToolchainService toolChainService;
 
     @Inject
     public GlobalBuildInfoPlugin(
+        ObjectFactory objectFactory,
         JavaInstallationRegistry javaInstallationRegistry,
         JvmMetadataDetector metadataDetector,
         ProviderFactory providers
     ) {
+        this.objectFactory = objectFactory;
         this.javaInstallationRegistry = javaInstallationRegistry;
         this.metadataDetector = new ErrorTraceMetadataDetector(metadataDetector);
         this.providers = providers;
+
     }
 
     @Override
     public void apply(Project project) {
         if (project != project.getRootProject()) {
             throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project.");
         }
+        project.getPlugins().apply(JvmToolchainsPlugin.class);
+        toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
         GradleVersion minimumGradleVersion = GradleVersion.version(getResourceContents("/minimumGradleVersion"));
         if (GradleVersion.current().compareTo(minimumGradleVersion) < 0) {
             throw new GradleException("Gradle " + minimumGradleVersion.getVersion() + "+ is required");
@@ -114,6 +127,8 @@ public void apply(Project project) {
             params.setTestSeed(getTestSeed());
             params.setIsCi(System.getenv("JENKINS_URL") != null || System.getenv("BUILDKITE_BUILD_URL") != null);
             params.setDefaultParallel(ParallelDetector.findDefaultParallel(project));
+            // TODO: Test if cc issues are coming from here
+            params.setDefaultParallel(8);
             params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false));
             params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true));
             AtomicReference<BwcVersions> cache = new AtomicReference<>();
@@ -219,7 +234,7 @@ private InstallationLocation getJavaInstallation(File javaHome) {
 
     private boolean isSameFile(File javaHome, InstallationLocation installationLocation) {
         try {
-            return Files.isSameFile(installationLocation.getLocation().toPath(), javaHome.toPath());
+            return Files.isSameFile(javaHome.toPath(), installationLocation.getLocation().toPath());
         } catch (IOException ioException) {
             throw new UncheckedIOException(ioException);
         }
@@ -294,9 +309,14 @@ private File findRuntimeJavaHome() {
     }
 
     private String findJavaHome(String version) {
-        Provider<String> javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv");
         String javaHomeEnvVar = getJavaHomeEnvVarName(version);
+        String env = System.getenv(javaHomeEnvVar);
+        return env != null ? resolveJavaHomeFromEnvVariable(javaHomeEnvVar) : resolveJavaHomeFromToolChainService(version);
+    }
 
+    @NotNull
+    private String resolveJavaHomeFromEnvVariable(String javaHomeEnvVar) {
+        Provider<String> javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv");
         // Provide a useful error if we're looking for a Java home version that we haven't told Gradle about yet
         Arrays.stream(javaHomeNames.get().split(","))
             .filter(s -> s.equals(javaHomeEnvVar))
@@ -309,7 +329,6 @@ private String findJavaHome(String version) {
                         + "updated in gradle.properties file."
                 )
             );
-
         String versionedJavaHome = System.getenv(javaHomeEnvVar);
         if (versionedJavaHome == null) {
             final String exceptionMessage = String.format(
@@ -320,12 +339,25 @@ private String findJavaHome(String version) {
                     + "it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details.",
                 javaHomeEnvVar
             );
-
             throw new GradleException(exceptionMessage);
         }
         return versionedJavaHome;
     }
 
+    @NotNull
+    private String resolveJavaHomeFromToolChainService(String version) {
+        Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
+        Provider<JavaLauncher> javaLauncherProvider = toolChainService.launcherFor(
+            javaToolchainSpec -> javaToolchainSpec.getLanguageVersion().value(value)
+        );
+
+        try {
+            return javaLauncherProvider.get().getMetadata().getInstallationPath().getAsFile().getCanonicalPath();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     private static String getJavaHomeEnvVarName(String version) {
         return "JAVA" + version + "_HOME";
     }
@@ -369,7 +401,7 @@ private static class MetadataBasedToolChainMatcher implements Action<JavaToolcha
         private final JvmVendorSpec expectedVendorSpec;
         private final JavaLanguageVersion expectedJavaLanguageVersion;
 
-        public MetadataBasedToolChainMatcher(JvmInstallationMetadata metadata) {
+        MetadataBasedToolChainMatcher(JvmInstallationMetadata metadata) {
             expectedVendorSpec = JvmVendorSpec.matching(metadata.getVendor().getRawVendor());
             expectedJavaLanguageVersion = JavaLanguageVersion.of(metadata.getLanguageVersion().getMajorVersion());
         }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AbstractCustomJavaToolchainResolver.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.toolchain;
+
+import org.gradle.jvm.toolchain.JavaToolchainResolver;
+import org.gradle.jvm.toolchain.JvmVendorSpec;
+import org.gradle.platform.Architecture;
+import org.gradle.platform.OperatingSystem;
+
+abstract class AbstractCustomJavaToolchainResolver implements JavaToolchainResolver {
+
+    static String toOsString(OperatingSystem operatingSystem) {
+        return toOsString(operatingSystem, null);
+    }
+
+    static String toOsString(OperatingSystem operatingSystem, JvmVendorSpec v) {
+        return switch (operatingSystem) {
+            case MAC_OS -> (v == null || v.equals(JvmVendorSpec.ADOPTIUM) == false) ? "macos" : "mac";
+            case LINUX -> "linux";
+            case WINDOWS -> "windows";
+            default -> throw new UnsupportedOperationException("Operating system " + operatingSystem);
+        };
+    }
+
+    static String toArchString(Architecture architecture) {
+        return switch (architecture) {
+            case X86_64 -> "x64";
+            case AARCH64 -> "aarch64";
+            case X86 -> "x86";
+        };
+    }
+
+    protected static boolean anyVendorOr(JvmVendorSpec givenVendor, JvmVendorSpec expectedVendor) {
+        return givenVendor.matches("any") || givenVendor.equals(expectedVendor);
+    }
+}