Add JUnit rule based integration test cluster orchestration framework (…

…elastic#92379)

This commit adds a new test framework for configuring and orchestrating
test clusters for both Java and YAML REST testing. This will eventually
replace the existing "test-clusters" Gradle plugin and the build-time
cluster orchestration.

build-tools-internal/build.gradle

@@ -115,6 +115,10 @@ gradlePlugin {
       id = 'elasticsearch.java'
       implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchJavaPlugin'
     }
+    legacyInternalJavaRestTest {
+      id = 'elasticsearch.legacy-java-rest-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.LegacyJavaRestTestPlugin'
+    }
     internalJavaRestTest {
       id = 'elasticsearch.internal-java-rest-test'
       implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin'
@@ -167,9 +171,17 @@ gradlePlugin {
       id = 'elasticsearch.validate-rest-spec'
       implementationClass = 'org.elasticsearch.gradle.internal.precommit.ValidateRestSpecPlugin'
     }
+    legacyYamlRestCompatTest {
+      id = 'elasticsearch.legacy-yaml-rest-compat-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.compat.compat.LegacyYamlRestCompatTestPlugin'
+    }
     yamlRestCompatTest {
       id = 'elasticsearch.yaml-rest-compat-test'
-      implementationClass = 'org.elasticsearch.gradle.internal.rest.compat.YamlRestCompatTestPlugin'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.compat.compat.YamlRestCompatTestPlugin'
+    }
+    legacyYamlRestTest {
+      id = 'elasticsearch.legacy-yaml-rest-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin'
     }
     yamlRestTest {
       id = 'elasticsearch.internal-yaml-rest-test'

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy

@@ -13,6 +13,7 @@ abstract class AbstractRestResourcesFuncTest extends AbstractGradleFuncTest {
 
     def setup() {
         subProject(":test:framework") << "apply plugin: 'elasticsearch.java'"
+        subProject(":test:test-clusters") << "apply plugin: 'elasticsearch.java'"
         subProject(":test:yaml-rest-runner") << "apply plugin: 'elasticsearch.java'"
 
         subProject(":rest-api-spec") << """

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy

@@ -174,12 +174,12 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPl
         given:
         clazz(dir('src/yamlRestTest/java'), "org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase")
         buildFile << """
-        apply plugin:'elasticsearch.internal-yaml-rest-test'
-        
+        apply plugin:'elasticsearch.legacy-yaml-rest-test'
+
         dependencies {
             yamlRestTestImplementation "org.apache.lucene:tests.util:1.0"
             yamlRestTestImplementation "org.junit:junit:4.42"
-        }    
+        }
         """
 
         clazz(dir("src/yamlRestTest/java"), "org.acme.valid.SomeMatchingIT", "org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase") {
@@ -216,11 +216,11 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPl
         buildFile << """
         import org.elasticsearch.gradle.internal.precommit.TestingConventionsCheckTask
         apply plugin:'$pluginName'
-        
+
         dependencies {
             ${sourceSetName}Implementation "org.apache.lucene:tests.util:1.0"
             ${sourceSetName}Implementation "org.junit:junit:4.42"
-        }    
+        }
         tasks.withType(TestingConventionsCheckTask).configureEach {
             suffix 'IT'
             suffix 'Tests'
@@ -252,19 +252,19 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPl
         )
 
         where:
-        pluginName                              | taskName                                | sourceSetName
-        "elasticsearch.internal-java-rest-test" | ":javaRestTestTestingConventions"       | "javaRestTest"
+        pluginName                              | taskName                                 | sourceSetName
+        "elasticsearch.legacy-java-rest-test"   | ":javaRestTestTestingConventions"        | "javaRestTest"
         "elasticsearch.internal-cluster-test"   | ":internalClusterTestTestingConventions" | "internalClusterTest"
     }
 
     private void simpleJavaBuild() {
         buildFile << """
         apply plugin:'java'
-                
+
         dependencies {
             testImplementation "org.apache.lucene:tests.util:1.0"
             testImplementation "org.junit:junit:4.42"
-        }    
+        }
         """
     }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy → build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy

@@ -17,7 +17,7 @@ import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
 import org.elasticsearch.gradle.VersionProperties
 import org.gradle.testkit.runner.TaskOutcome
 
-class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
+class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
     def compatibleVersion = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1
     def specIntermediateDir = "restResources/v${compatibleVersion}/yamlSpecs"
@@ -45,7 +45,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
         buildFile << """
         plugins {
-          id 'elasticsearch.yaml-rest-compat-test'
+          id 'elasticsearch.legacy-yaml-rest-compat-test'
         }
         """
 
@@ -71,7 +71,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         """
 
         buildFile << """
-            apply plugin: 'elasticsearch.yaml-rest-compat-test'
+            apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
             // avoids a dependency problem in this test, the distribution in use here is inconsequential to the test
             import org.elasticsearch.gradle.testclusters.TestDistribution;
@@ -151,7 +151,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
         buildFile << """
         plugins {
-          id 'elasticsearch.yaml-rest-compat-test'
+          id 'elasticsearch.legacy-yaml-rest-compat-test'
         }
 
         """
@@ -194,7 +194,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         """
 
         buildFile << """
-            apply plugin: 'elasticsearch.yaml-rest-compat-test'
+            apply plugin: 'elasticsearch.legacy-yaml-rest-compat-test'
 
             // avoids a dependency problem in this test, the distribution in use here is inconsequential to the test
             import org.elasticsearch.gradle.testclusters.TestDistribution;

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPluginFuncTest.groovy → build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestTestPluginFuncTest.groovy

@@ -15,15 +15,15 @@ import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
 import org.gradle.testkit.runner.TaskOutcome
 
 @IgnoreIf({ os.isWindows() })
-class InternalYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
+class LegacyYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
     def "yamlRestTest does nothing when there are no tests"() {
         given:
         // RestIntegTestTask not cc compatible due to
         configurationCacheCompatible = false
         buildFile << """
         plugins {
-          id 'elasticsearch.internal-yaml-rest-test'
+          id 'elasticsearch.legacy-yaml-rest-test'
         }
         """
 
@@ -42,7 +42,7 @@ class InternalYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         configurationCacheCompatible = false
         internalBuild()
         buildFile << """
-            apply plugin: 'elasticsearch.internal-yaml-rest-test'
+            apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
             dependencies {
                yamlRestTestImplementation "junit:junit:4.12"
@@ -96,7 +96,7 @@ class InternalYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         def subProjectBuildFile = subProject(pluginProjectPath)
         subProjectBuildFile << """
             apply plugin: 'elasticsearch.esplugin'
-            apply plugin: 'elasticsearch.internal-yaml-rest-test'
+            apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 
             dependencies {
                yamlRestTestImplementation "junit:junit:4.12"

build-tools-internal/src/main/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPlugin.groovy

@@ -38,7 +38,7 @@ class DocsTestPlugin implements Plugin<Project> {
 
     @Override
     void apply(Project project) {
-        project.pluginManager.apply('elasticsearch.internal-yaml-rest-test')
+        project.pluginManager.apply('elasticsearch.legacy-yaml-rest-test')
 
         String distribution = System.getProperty('tests.distribution', 'default')
         // The distribution can be configured with -Dtests.distribution on the command line

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ResolveAllDependencies.java

@@ -21,7 +21,7 @@
 import javax.inject.Inject;
 
 import static org.elasticsearch.gradle.DistributionDownloadPlugin.DISTRO_EXTRACTED_CONFIG_PREFIX;
-import static org.elasticsearch.gradle.internal.rest.compat.YamlRestCompatTestPlugin.BWC_MINOR_CONFIG_NAME;
+import static org.elasticsearch.gradle.internal.test.rest.compat.compat.LegacyYamlRestCompatTestPlugin.BWC_MINOR_CONFIG_NAME;
 
 public class ResolveAllDependencies extends DefaultTask {
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPlugin.java

@@ -11,7 +11,8 @@
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
 import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin;
 import org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin;
-import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.LegacyJavaRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin;
 import org.gradle.api.Action;
 import org.gradle.api.NamedDomainObjectProvider;
 import org.gradle.api.Project;
@@ -43,8 +44,8 @@ public TaskProvider<? extends Task> createTask(Project project) {
             });
         });
 
-        project.getPlugins().withType(InternalYamlRestTestPlugin.class, yamlRestTestPlugin -> {
-            NamedDomainObjectProvider<SourceSet> sourceSet = sourceSets.named(InternalYamlRestTestPlugin.SOURCE_SET_NAME);
+        project.getPlugins().withType(LegacyYamlRestTestPlugin.class, yamlRestTestPlugin -> {
+            NamedDomainObjectProvider<SourceSet> sourceSet = sourceSets.named(LegacyYamlRestTestPlugin.SOURCE_SET_NAME);
             setupTaskForSourceSet(project, sourceSet, t -> {
                 t.getSuffixes().convention(List.of("IT"));
                 t.getBaseClasses().convention(List.of("org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase"));
@@ -68,8 +69,17 @@ public TaskProvider<? extends Task> createTask(Project project) {
             });
         });
 
+        project.getPlugins().withType(LegacyJavaRestTestPlugin.class, javaRestTestPlugin -> {
+            NamedDomainObjectProvider<SourceSet> sourceSet = sourceSets.named(LegacyJavaRestTestPlugin.SOURCE_SET_NAME);
+            setupTaskForSourceSet(project, sourceSet, t -> {
+                t.getSuffixes().convention(List.of("IT"));
+                t.getBaseClasses()
+                    .convention(List.of("org.elasticsearch.test.ESIntegTestCase", "org.elasticsearch.test.rest.ESRestTestCase"));
+            });
+        });
+
         project.getPlugins().withType(InternalJavaRestTestPlugin.class, javaRestTestPlugin -> {
-            NamedDomainObjectProvider<SourceSet> sourceSet = sourceSets.named(InternalJavaRestTestPlugin.SOURCE_SET_NAME);
+            NamedDomainObjectProvider<SourceSet> sourceSet = sourceSets.named(LegacyJavaRestTestPlugin.SOURCE_SET_NAME);
             setupTaskForSourceSet(project, sourceSet, t -> {
                 t.getSuffixes().convention(List.of("IT"));
                 t.getBaseClasses()

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestTestBasePlugin.java → build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java

@@ -31,7 +31,11 @@
 import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.BUNDLE_PLUGIN_TASK_NAME;
 import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.EXPLODED_BUNDLE_PLUGIN_TASK_NAME;
 
-public class RestTestBasePlugin implements Plugin<Project> {
+/**
+ * @deprecated use {@link RestTestBasePlugin} instead
+ */
+@Deprecated
+public class LegacyRestTestBasePlugin implements Plugin<Project> {
     private static final String TESTS_REST_CLUSTER = "tests.rest.cluster";
     private static final String TESTS_CLUSTER = "tests.cluster";
     private static final String TESTS_CLUSTER_NAME = "tests.clustername";
@@ -40,7 +44,7 @@ public class RestTestBasePlugin implements Plugin<Project> {
     private ProviderFactory providerFactory;
 
     @Inject
-    public RestTestBasePlugin(ProviderFactory providerFactory) {
+    public LegacyRestTestBasePlugin(ProviderFactory providerFactory) {
         this.providerFactory = providerFactory;
     }
 
@@ -87,17 +91,19 @@ public void apply(Project project) {
             .withType(StandaloneRestIntegTestTask.class)
             .configureEach(t -> t.finalizedBy(project.getTasks().withType(FixtureStop.class)));
 
-        project.getTasks().withType(StandaloneRestIntegTestTask.class).configureEach(t ->
-        // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster
-        project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
-            if (GradleUtils.isModuleProject(project.getPath())) {
-                var bundle = project.getTasks().withType(Sync.class).named(EXPLODED_BUNDLE_PLUGIN_TASK_NAME);
-                t.getClusters().forEach(c -> c.module(bundle));
-            } else {
-                var bundle = project.getTasks().withType(Zip.class).named(BUNDLE_PLUGIN_TASK_NAME);
-                t.getClusters().forEach(c -> c.plugin(bundle));
-            }
-        }));
+        project.getTasks().withType(StandaloneRestIntegTestTask.class).configureEach(t -> {
+            t.setMaxParallelForks(1);
+            // if this a module or plugin, it may have an associated zip file with it's contents, add that to the test cluster
+            project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
+                if (GradleUtils.isModuleProject(project.getPath())) {
+                    var bundle = project.getTasks().withType(Sync.class).named(EXPLODED_BUNDLE_PLUGIN_TASK_NAME);
+                    t.getClusters().forEach(c -> c.module(bundle));
+                } else {
+                    var bundle = project.getTasks().withType(Zip.class).named(BUNDLE_PLUGIN_TASK_NAME);
+                    t.getClusters().forEach(c -> c.plugin(bundle));
+                }
+            });
+        });
     }
 
     private String systemProperty(String propName) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java

@@ -11,8 +11,8 @@
 import org.elasticsearch.gradle.internal.ExportElasticsearchBuildResourcesTask;
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
 import org.elasticsearch.gradle.internal.precommit.InternalPrecommitTasks;
-import org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin;
-import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.LegacyJavaRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.LegacyYamlRestTestPlugin;
 import org.elasticsearch.gradle.internal.test.rest.RestTestUtil;
 import org.gradle.api.InvalidUserDataException;
 import org.gradle.api.Plugin;
@@ -32,8 +32,8 @@
  * and run REST tests. Use BuildPlugin if you want to build main code as well
  * as tests.
  *
- * @deprecated use {@link InternalClusterTestPlugin}, {@link InternalJavaRestTestPlugin} or
- * {@link InternalYamlRestTestPlugin} instead.
+ * @deprecated use {@link InternalClusterTestPlugin}, {@link LegacyJavaRestTestPlugin} or
+ * {@link LegacyYamlRestTestPlugin} instead.
  */
 @Deprecated
 public class StandaloneRestTestPlugin implements Plugin<Project> {
@@ -46,7 +46,7 @@ public void apply(final Project project) {
         }
 
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
-        project.getPluginManager().apply(RestTestBasePlugin.class);
+        project.getPluginManager().apply(LegacyRestTestBasePlugin.class);
 
         project.getTasks().register("buildResources", ExportElasticsearchBuildResourcesTask.class);
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java

@@ -14,7 +14,7 @@
 import org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin;
 import org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin;
 import org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsTask;
-import org.elasticsearch.gradle.internal.test.rest.InternalJavaRestTestPlugin;
+import org.elasticsearch.gradle.internal.test.rest.LegacyJavaRestTestPlugin;
 import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
 import org.elasticsearch.gradle.testclusters.TestClustersAware;
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
@@ -62,8 +62,8 @@ public void apply(Project project) {
                 .withType(RestIntegTestTask.class)
                 .configureEach(runner -> runner.systemProperty("tests.ssl.enabled", "true"));
         });
-        project.getPlugins().withType(InternalJavaRestTestPlugin.class).configureEach(restTestPlugin -> {
-            SourceSet testSourceSet = Util.getJavaSourceSets(project).getByName(InternalJavaRestTestPlugin.SOURCE_SET_NAME);
+        project.getPlugins().withType(LegacyJavaRestTestPlugin.class).configureEach(restTestPlugin -> {
+            SourceSet testSourceSet = Util.getJavaSourceSets(project).getByName(LegacyJavaRestTestPlugin.SOURCE_SET_NAME);
             testSourceSet.getResources().srcDir(new File(keyStoreDir, "test/ssl"));
             project.getTasks().named(testSourceSet.getProcessResourcesTaskName()).configure(t -> t.dependsOn(exportKeyStore));
             project.getTasks().withType(TestClustersAware.class).configureEach(clusterAware -> clusterAware.dependsOn(exportKeyStore));

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalJavaRestTestPlugin.java

@@ -8,7 +8,7 @@
 
 package org.elasticsearch.gradle.internal.test.rest;
 
-import org.elasticsearch.gradle.internal.test.RestTestBasePlugin;
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -33,8 +33,11 @@ public void apply(Project project) {
         SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
         SourceSet javaTestSourceSet = sourceSets.create(SOURCE_SET_NAME);
 
+        project.getDependencies().add(javaTestSourceSet.getImplementationConfigurationName(), project.project(":test:test-clusters"));
+
         // setup the javaRestTest task
-        registerTestTask(project, javaTestSourceSet);
+        // we use a StandloneRestIntegTestTask here so that the conventions of RestTestBasePlugin don't create a test cluster
+        registerTestTask(project, javaTestSourceSet, SOURCE_SET_NAME, StandaloneRestIntegTestTask.class);
 
         // setup dependencies
         setupJavaRestTestDependenciesDefaults(project, javaTestSourceSet);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPlugin.java

@@ -8,7 +8,7 @@
 
 package org.elasticsearch.gradle.internal.test.rest;
 
-import org.elasticsearch.gradle.internal.test.RestTestBasePlugin;
+import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -34,10 +34,10 @@ public void apply(Project project) {
         SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
         SourceSet yamlTestSourceSet = sourceSets.create(SOURCE_SET_NAME);
 
-        registerTestTask(project, yamlTestSourceSet);
+        registerTestTask(project, yamlTestSourceSet, SOURCE_SET_NAME, StandaloneRestIntegTestTask.class);
 
         // setup the dependencies
-        setupYamlRestTestDependenciesDefaults(project, yamlTestSourceSet);
+        setupYamlRestTestDependenciesDefaults(project, yamlTestSourceSet, true);
 
         // setup the copy for the rest resources
         project.getTasks().withType(CopyRestApiTask.class).configureEach(copyRestApiTask -> {