Skip to content

Commit

Permalink
Refactor global build info plugin to leverage JavaInstallationRegistr…
Browse files Browse the repository at this point in the history 
…y (#54026)

This commit removes the configuration time vs execution time distinction
with regards to certain BuildParms properties. Because of the cost of
determining Java versions for configuration JDK locations we deferred
this until execution time. This had two main downsides. First, we had
to implement all this build logic in tasks, which required a bunch of
additional plumbing and complexity. Second, because some information
wasn't known during configuration time, we had to nest any build logic
that depended on this in awkward callbacks.

We now defer to the JavaInstallationRegistry recently added in Gradle.
This utility uses a much more efficient method for probing Java
installations vs our jrunscript implementation. This, combined with some
optimizations to avoid probing the current JVM as well as deferring
some evaluation via Providers when probing installations for BWC builds
we can maintain effectively the same configuration time performance
while removing a bunch of complexity and runtime cost (snapshotting
inputs for the GenerateGlobalBuildInfoTask was very expensive). The end
result should be a much more responsive build execution in almost all
scenarios.

(cherry picked from commit ecdbd37f2e0f0447ed574b306adb64c19adc3ce1)
  • Loading branch information
@mark-vieira
mark-vieira authored Mar 23, 2020
1 parent c97ee4e commit 70cfedf
Show file tree
Hide file tree
Showing 34 changed files with 646 additions and 1,024 deletions.
3 changes: 0 additions & 3 deletions CONTRIBUTING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,6 @@ be used to test against other JDKs as well, this is not only limited to JDK 8.
> Note: It is also required to have `JAVA8_HOME`, `JAVA9_HOME`, `JAVA10_HOME`
and `JAVA11_HOME`, and `JAVA12_HOME` available so that the tests can pass.

> Warning: do not use `sdkman` for Java installations which do not have proper
`jrunscript` for jdk distributions.

Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
using the wrapper via the `gradlew` script on Unix systems or `gradlew.bat`
script on Windows in the root of the repository. The examples below show the
Expand Down
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder
import org.gradle.util.DistributionLocator
import org.gradle.util.GradleVersion

import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure

plugins {
id 'lifecycle-base'
Expand Down
84 changes: 35 additions & 49 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,15 @@ import groovy.transform.CompileStatic
import org.apache.commons.io.IOUtils
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
import org.elasticsearch.gradle.info.GlobalInfoExtension
import org.elasticsearch.gradle.info.JavaHome
import org.elasticsearch.gradle.precommit.DependencyLicensesTask
import org.elasticsearch.gradle.precommit.PrecommitTasks
import org.elasticsearch.gradle.test.ErrorReportingTestListener
import org.elasticsearch.gradle.testclusters.ElasticsearchCluster
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
import org.gradle.api.Action
import org.elasticsearch.gradle.testclusters.TestDistribution
import org.elasticsearch.gradle.tool.Boilerplate
import org.elasticsearch.gradle.util.GradleUtils
import org.gradle.api.Action
import org.gradle.api.GradleException
import org.gradle.api.InvalidUserDataException
import org.gradle.api.JavaVersion
Expand Down Expand Up @@ -84,7 +83,7 @@ import org.gradle.util.GradleVersion
import java.nio.charset.StandardCharsets
import java.nio.file.Files

import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure

/**
* Encapsulates build configuration for elasticsearch projects.
Expand Down Expand Up @@ -146,7 +145,7 @@ class BuildPlugin implements Plugin<Project> {
// Common config when running with a FIPS-140 runtime JVM
if (inFipsJvm()) {
// This configuration can be removed once system modules are available
Boilerplate.maybeCreate(project.configurations, 'extraJars') {
GradleUtils.maybeCreate(project.configurations, 'extraJars') {
project.dependencies.add('extraJars', "org.bouncycastle:bc-fips:1.0.1")
project.dependencies.add('extraJars', "org.bouncycastle:bctls-fips:1.0.9")
}
Expand All @@ -156,7 +155,6 @@ class BuildPlugin implements Plugin<Project> {
File securityPolicy = buildResources.copy("fips_java.policy")
File security8Policy = buildResources.copy("fips_java8.policy")
File bcfksKeystore = buildResources.copy("cacerts.bcfks")
GlobalInfoExtension globalInfo = project.rootProject.extensions.getByType(GlobalInfoExtension)
project.pluginManager.withPlugin("elasticsearch.testclusters") {
NamedDomainObjectContainer<ElasticsearchCluster> testClusters = project.extensions.findByName(TestClustersPlugin.EXTENSION_NAME) as NamedDomainObjectContainer<ElasticsearchCluster>
if (testClusters != null) {
Expand All @@ -165,14 +163,12 @@ class BuildPlugin implements Plugin<Project> {
for (File dep : project.getConfigurations().getByName("extraJars").getFiles()) {
cluster.extraJarFile(dep)
}
globalInfo.ready {
if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
cluster.extraConfigFile("fips_java.security", securityProperties)
cluster.extraConfigFile("fips_java.policy", securityPolicy)
} else {
cluster.extraConfigFile("fips_java.security", security8Properties)
cluster.extraConfigFile("fips_java.policy", security8Policy)
}
if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
cluster.extraConfigFile("fips_java.security", securityProperties)
cluster.extraConfigFile("fips_java.policy", securityPolicy)
} else {
cluster.extraConfigFile("fips_java.security", security8Properties)
cluster.extraConfigFile("fips_java.policy", security8Policy)
}
cluster.extraConfigFile("cacerts.bcfks", bcfksKeystore)
cluster.systemProperty('java.security.properties', '=${ES_PATH_CONF}/fips_java.security')
Expand All @@ -188,16 +184,14 @@ class BuildPlugin implements Plugin<Project> {
}
project.tasks.withType(Test).configureEach { Test task ->
task.dependsOn(buildResources)
globalInfo.ready {
// Using the key==value format to override default JVM security settings and policy
// see also: https://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html
if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
task.systemProperty('java.security.properties', String.format(Locale.ROOT, "=%s", securityProperties.toString()))
task.systemProperty('java.security.policy', String.format(Locale.ROOT, "=%s", securityPolicy.toString()))
} else {
task.systemProperty('java.security.properties', String.format(Locale.ROOT, "=%s", security8Properties.toString()))
task.systemProperty('java.security.policy', String.format(Locale.ROOT, "=%s", security8Policy.toString()))
}
// Using the key==value format to override default JVM security settings and policy
// see also: https://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html
if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
task.systemProperty('java.security.properties', String.format(Locale.ROOT, "=%s", securityProperties.toString()))
task.systemProperty('java.security.policy', String.format(Locale.ROOT, "=%s", securityPolicy.toString()))
} else {
task.systemProperty('java.security.properties', String.format(Locale.ROOT, "=%s", security8Properties.toString()))
task.systemProperty('java.security.policy', String.format(Locale.ROOT, "=%s", security8Policy.toString()))
}
task.systemProperty('javax.net.ssl.trustStorePassword', 'password')
task.systemProperty('javax.net.ssl.keyStorePassword', 'password')
Expand Down Expand Up @@ -258,7 +252,7 @@ class BuildPlugin implements Plugin<Project> {
static String getJavaHome(final Task task, final int version) {
requireJavaHome(task, version)
JavaHome java = BuildParams.javaVersions.find { it.version == version }
return java == null ? null : java.javaHome.absolutePath
return java == null ? null : java.javaHome.get().absolutePath
}

/**
Expand Down Expand Up @@ -407,7 +401,7 @@ class BuildPlugin implements Plugin<Project> {
dependencyNode.appendNode('groupId', dependency.group)
dependencyNode.appendNode('artifactId', dependency.getDependencyProject().convention.getPlugin(BasePluginConvention).archivesBaseName)
dependencyNode.appendNode('version', dependency.version)
dependencyNode.appendNode('scope', 'runtime')
dependencyNode.appendNode('scope', 'compile')
}
}
}
Expand All @@ -426,13 +420,10 @@ class BuildPlugin implements Plugin<Project> {
/** Adds compiler settings to the project */
static void configureCompile(Project project) {
ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension)
GlobalInfoExtension globalBuildInfo = project.rootProject.extensions.getByType(GlobalInfoExtension)
globalBuildInfo.ready {
if (BuildParams.compilerJavaVersion < JavaVersion.VERSION_1_10) {
ext.set('compactProfile', 'compact3')
} else {
ext.set('compactProfile', 'full')
}
if (BuildParams.compilerJavaVersion < JavaVersion.VERSION_1_10) {
ext.set('compactProfile', 'compact3')
} else {
ext.set('compactProfile', 'full')
}
ext.set('compactProfile', 'full')

Expand All @@ -450,12 +441,10 @@ class BuildPlugin implements Plugin<Project> {
compileTask.options.forkOptions.javaHome = BuildParams.compilerJavaHome
}
if (targetCompatibilityVersion == JavaVersion.VERSION_1_8) {
globalBuildInfo.ready {
// compile with compact 3 profile by default
// NOTE: this is just a compile time check: does not replace testing with a compact3 JRE
if (ext.get('compactProfile') != 'full') {
compileTask.options.compilerArgs << '-profile' << ext.get('compactProfile').toString()
}
// compile with compact 3 profile by default
// NOTE: this is just a compile time check: does not replace testing with a compact3 JRE
if (ext.get('compactProfile') != 'full') {
compileTask.options.compilerArgs << '-profile' << ext.get('compactProfile').toString()
}
}
/*
Expand Down Expand Up @@ -657,16 +646,13 @@ class BuildPlugin implements Plugin<Project> {
project.mkdir(heapdumpDir)
project.mkdir(test.workingDir)
project.mkdir(test.workingDir.toPath().resolve('temp'))

if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_1_9) {
test.jvmArgs '--illegal-access=warn'
}
//TODO remove once jvm.options are added to test system properties
if (BuildParams.runtimeJavaVersion == JavaVersion.VERSION_1_8) {
test.systemProperty ('java.locale.providers','SPI,JRE')
} else if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_1_9) {
test.systemProperty ('java.locale.providers','SPI,COMPAT')
}
}
//TODO remove once jvm.options are added to test system properties
if (BuildParams.runtimeJavaVersion == JavaVersion.VERSION_1_8) {
test.systemProperty ('java.locale.providers','SPI,JRE')
} else if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_1_9) {
test.systemProperty ('java.locale.providers','SPI,COMPAT')
test.jvmArgs '--illegal-access=warn'
}
if (inFipsJvm()) {
project.dependencies.add('testRuntimeOnly', "org.bouncycastle:bc-fips:1.0.1")
Expand Down
17 changes: 7 additions & 10 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,16 +143,13 @@ class PrecommitTasks {
ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
project.tasks.withType(CheckForbiddenApis).configureEach {
dependsOn(buildResources)
doFirst {
// we need to defer this configuration since we don't know the runtime java version until execution time
targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_13) {
project.logger.warn(
"Forbidden APIs does not support Java versions past 13. Will use the signatures from 13 for {}.",
BuildParams.runtimeJavaVersion
)
targetCompatibility = JavaVersion.VERSION_13.majorVersion
}
targetCompatibility = BuildParams.runtimeJavaVersion.majorVersion
if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_13) {
project.logger.warn(
"Forbidden APIs does not support Java versions past 13. Will use the signatures from 13 for {}.",
BuildParams.runtimeJavaVersion
)
targetCompatibility = JavaVersion.VERSION_13.majorVersion
}
bundledSignatures = [
"jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"
Expand Down
6 changes: 3 additions & 3 deletions buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import org.elasticsearch.gradle.docker.DockerSupportService;
import org.elasticsearch.gradle.info.BuildParams;
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
import org.elasticsearch.gradle.tool.Boilerplate;
import org.elasticsearch.gradle.util.GradleUtils;
import org.gradle.api.GradleException;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin;
Expand All @@ -53,7 +53,7 @@
import java.util.concurrent.Callable;
import java.util.function.Supplier;

import static org.elasticsearch.gradle.Util.capitalize;
import static org.elasticsearch.gradle.util.Util.capitalize;

/**
* A plugin to manage getting and extracting distributions of Elasticsearch.
Expand All @@ -78,7 +78,7 @@ public void apply(Project project) {
project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class);

Provider<DockerSupportService> dockerSupport = Boilerplate.getBuildService(
Provider<DockerSupportService> dockerSupport = GradleUtils.getBuildService(
project.getGradle().getSharedServices(),
DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
);
Expand Down
4 changes: 2 additions & 2 deletions buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@
import java.util.concurrent.Callable;
import java.util.stream.StreamSupport;

import static org.elasticsearch.gradle.tool.Boilerplate.findByName;
import static org.elasticsearch.gradle.tool.Boilerplate.maybeCreate;
import static org.elasticsearch.gradle.util.GradleUtils.findByName;
import static org.elasticsearch.gradle.util.GradleUtils.maybeCreate;

public class JdkDownloadPlugin implements Plugin<Project> {

Expand Down
Loading

0 comments on commit 70cfedf

Please sign in to comment.