Port idp-fixture to testcontainers (elastic#103320)

This ports idp-fixture to test container and updates downstream tests accordingly.
natasha-moore-elastic · Dec 13, 2023 · 6e36ea8 · 6e36ea8
1 parent 40b817e
commit 6e36ea8
Show file tree

Hide file tree

Showing 156 changed files with 629 additions and 247 deletions.
diff --git a/...al/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java b/...al/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java
@@ -25,7 +25,9 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.Constructor;
 import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
@@ -74,9 +76,14 @@ public void execute() {
                 Set<Class<?>> classes = (Set<Class<?>>) reflections.getSubTypesOf(ifClass);
 
                 for (Class<?> cacheableTestFixtureClazz : classes) {
-                    Object o = cacheableTestFixtureClazz.getDeclaredConstructor().newInstance();
-                    Method cacheMethod = cacheableTestFixtureClazz.getMethod("cache");
-                    cacheMethod.invoke(o);
+                    if (Modifier.isAbstract(cacheableTestFixtureClazz.getModifiers()) == false) {
+                        Constructor<?> declaredConstructor = cacheableTestFixtureClazz.getDeclaredConstructor();
+                        declaredConstructor.setAccessible(true);
+                        Object o = declaredConstructor.newInstance();
+                        Method cacheMethod = cacheableTestFixtureClazz.getMethod("cache");
+                        System.out.println("Caching resources from " + cacheableTestFixtureClazz.getName());
+                        cacheMethod.invoke(o);
+                    }
                 }
             } catch (Exception e) {
                 throw new RuntimeException(e);

diff --git a/docs/build.gradle b/docs/build.gradle
@@ -113,8 +113,9 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
   requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0")
 
-  extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("oidc/op-jwks.json")
-  extraConfigFile 'idp-docs-metadata.xml', project(':x-pack:test:idp-fixture').file("idp/shibboleth-idp/metadata/idp-docs-metadata.xml")
+  // TODO Rene: clean up this kind of cross project file references
+  extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("src/main/resources/oidc/op-jwks.json")
+  extraConfigFile 'idp-docs-metadata.xml', project(':x-pack:test:idp-fixture').file("src/main/resources/idp/shibboleth-idp/metadata/idp-docs-metadata.xml")
   extraConfigFile 'testClient.crt', project(':x-pack:plugin:security').file("src/test/resources/org/elasticsearch/xpack/security/action/pki_delegation/testClient.crt")
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.authc.api_key.enabled', 'true'

diff --git a/settings.gradle b/settings.gradle
@@ -97,6 +97,7 @@ List projects = [
   'test:fixtures:minio-fixture',
   'test:fixtures:old-elasticsearch',
   'test:fixtures:s3-fixture',
+  'test:fixtures:testcontainer-utils',
   'test:fixtures:geoip-fixture',
   'test:fixtures:url-fixture',
   'test:fixtures:nginx-fixture',

diff --git a/test/fixtures/minio-fixture/build.gradle b/test/fixtures/minio-fixture/build.gradle
@@ -5,7 +5,6 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-apply plugin: 'java'
 apply plugin: 'elasticsearch.java'
 apply plugin: 'elasticsearch.cache-test-fixtures'
 
@@ -19,6 +18,7 @@ dependencies {
   testImplementation project(':test:framework')
 
   api "junit:junit:${versions.junit}"
+  api project(':test:fixtures:testcontainer-utils')
   api "org.testcontainers:testcontainers:${versions.testcontainer}"
   implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   implementation "org.slf4j:slf4j-api:${versions.slf4j}"

diff --git a/...minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java b/...minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java
@@ -8,12 +8,10 @@
 
 package org.elasticsearch.test.fixtures.minio;
 
-import org.elasticsearch.test.fixtures.CacheableTestFixture;
 import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer;
-import org.junit.rules.TestRule;
 import org.testcontainers.images.builder.ImageFromDockerfile;
 
-public final class MinioTestContainer extends DockerEnvironmentAwareTestContainer implements TestRule, CacheableTestFixture {
+public final class MinioTestContainer extends DockerEnvironmentAwareTestContainer {
 
     private static final int servicePort = 9000;
     public static final String DOCKER_BASE_IMAGE = "minio/minio:RELEASE.2021-03-01T04-20-55Z";
@@ -25,7 +23,7 @@ public MinioTestContainer() {
 
     public MinioTestContainer(boolean enabled) {
         super(
-            new ImageFromDockerfile().withDockerfileFromBuilder(
+            new ImageFromDockerfile("es-minio-testfixture").withDockerfileFromBuilder(
                 builder -> builder.from(DOCKER_BASE_IMAGE)
                     .env("MINIO_ACCESS_KEY", "s3_test_access_key")
                     .env("MINIO_SECRET_KEY", "s3_test_secret_key")
@@ -50,13 +48,4 @@ public void start() {
     public String getAddress() {
         return "http://127.0.0.1:" + getMappedPort(servicePort);
     }
-
-    public void cache() {
-        try {
-            start();
-            stop();
-        } catch (RuntimeException e) {
-            logger().warn("Error while caching container images.", e);
-        }
-    }
 }
diff --git a/test/fixtures/testcontainer-utils/build.gradle b/test/fixtures/testcontainer-utils/build.gradle
@@ -0,0 +1,15 @@
+apply plugin: 'elasticsearch.java'
+
+
+configurations.all {
+  transitive = false
+}
+
+dependencies {
+  testImplementation project(':test:framework')
+  api "junit:junit:${versions.junit}"
+  api "org.testcontainers:testcontainers:${versions.testcontainer}"
+  implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  implementation "org.slf4j:slf4j-api:${versions.slf4j}"
+  implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}"
+}
diff --git a/...h/test/fixtures/CacheableTestFixture.java → ...h/test/fixtures/CacheableTestFixture.java b/...h/test/fixtures/CacheableTestFixture.java → ...h/test/fixtures/CacheableTestFixture.java
diff --git a/...ures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/ResourceUtils.java b/...ures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/ResourceUtils.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test.fixtures;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+
+public final class ResourceUtils {
+
+    public static Path copyResourceToFile(Class<?> clazz, Path targetFolder, String resourcePath) {
+        try {
+            ClassLoader classLoader = clazz.getClassLoader();
+            URL resourceUrl = classLoader.getResource(resourcePath);
+            if (resourceUrl == null) {
+                throw new RuntimeException("Failed to load " + resourcePath + " from classpath");
+            }
+            InputStream inputStream = resourceUrl.openStream();
+            File outputFile = new File(targetFolder.toFile(), resourcePath.substring(resourcePath.lastIndexOf("/")));
+            Files.copy(inputStream, outputFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+            return outputFile.toPath();
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to load ca.jks from classpath", e);
+        }
+    }
+}
diff --git a/.../DockerEnvironmentAwareTestContainer.java → .../DockerEnvironmentAwareTestContainer.java b/.../DockerEnvironmentAwareTestContainer.java → .../DockerEnvironmentAwareTestContainer.java
@@ -8,12 +8,14 @@
 
 package org.elasticsearch.test.fixtures.testcontainers;
 
-import org.elasticsearch.test.fixtures.minio.MinioTestContainer;
+import org.elasticsearch.test.fixtures.CacheableTestFixture;
 import org.junit.Assume;
+import org.junit.rules.TestRule;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
 import org.testcontainers.images.builder.ImageFromDockerfile;
 
 import java.io.File;
@@ -27,7 +29,10 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
-public class DockerEnvironmentAwareTestContainer extends GenericContainer<MinioTestContainer> {
+public abstract class DockerEnvironmentAwareTestContainer extends GenericContainer<DockerEnvironmentAwareTestContainer>
+    implements
+        TestRule,
+        CacheableTestFixture {
     protected static final Logger LOGGER = LoggerFactory.getLogger(DockerEnvironmentAwareTestContainer.class);
 
     private static final String DOCKER_ON_LINUX_EXCLUSIONS_FILE = ".ci/dockerOnLinuxExclusions";
@@ -53,6 +58,7 @@ private static boolean isDockerAvailable() {
 
     public DockerEnvironmentAwareTestContainer(ImageFromDockerfile imageFromDockerfile) {
         super(imageFromDockerfile);
+        withLogConsumer(new Slf4jLogConsumer(logger()));
     }
 
     @Override
@@ -62,6 +68,16 @@ public void start() {
         super.start();
     }
 
+    @Override
+    public void cache() {
+        try {
+            start();
+            stop();
+        } catch (RuntimeException e) {
+            logger().warn("Error while caching container images.", e);
+        }
+    }
+
     static String deriveId(Map<String, String> values) {
         return values.get("ID") + "-" + values.get("VERSION_ID");
     }

diff --git a/...ontainers/TestContainersThreadFilter.java → ...ontainers/TestContainersThreadFilter.java b/...ontainers/TestContainersThreadFilter.java → ...ontainers/TestContainersThreadFilter.java
@@ -17,6 +17,8 @@
 public class TestContainersThreadFilter implements ThreadFilter {
     @Override
     public boolean reject(Thread t) {
-        return t.getName().startsWith("testcontainers-") || t.getName().startsWith("ducttape");
+        return t.getName().startsWith("testcontainers-")
+            || t.getName().startsWith("ducttape")
+            || t.getName().startsWith("ForkJoinPool.commonPool-worker-1");
     }
 }
diff --git a/...ity/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java b/...ity/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java
@@ -54,6 +54,10 @@ protected static RealmConfig config(RealmConfig.RealmIdentifier realmId, Setting
 
     @Before
     public void setUpLdapConnection() throws Exception {
+        doSetupLdapConnection();
+    }
+
+    protected void doSetupLdapConnection() throws Exception {
         Path trustPath = getDataPath(trustPath());
         this.ldapConnection = LdapTestUtils.openConnection(ldapUrl(), bindDN(), bindPassword(), trustPath);
     }

diff --git a/x-pack/qa/oidc-op-tests/build.gradle b/x-pack/qa/oidc-op-tests/build.gradle
@@ -1,16 +1,13 @@
 import org.elasticsearch.gradle.internal.info.BuildParams
 
 apply plugin: 'elasticsearch.internal-java-rest-test'
-apply plugin: 'elasticsearch.test.fixtures'
 
 dependencies {
   javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
   javaRestTestImplementation(testArtifact(project(xpackModule('security'))))
+  javaRestTestImplementation project(":x-pack:test:idp-fixture")
 }
 
-testFixtures.useFixture ":x-pack:test:idp-fixture", "http-proxy"
-testFixtures.useFixture ":x-pack:test:idp-fixture", "oidc-provider"
-
 tasks.named('javaRestTest') {
   usesDefaultDistribution()
   // test suite uses jks which is not supported in fips mode

diff --git a/...s/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java b/...s/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java
@@ -106,7 +106,7 @@ public void testAuthenticateWithOidcIssuedJwt() throws Exception {
             new Scope(OIDCScopeValue.OPENID),
             new ClientID(clientId),
             new URI(redirectUri)
-        ).endpointURI(new URI(C2ID_AUTH_ENDPOINT)).state(new State(state)).nonce(new Nonce(nonce)).build();
+        ).endpointURI(new URI(c2id.getC2OPUrl() + "/c2id-login")).state(new State(state)).nonce(new Nonce(nonce)).build();
 
         final String implicitFlowURI = authenticateAtOP(oidcAuthRequest.toURI());