Port smb fixture from test fixture plugin to testcontainer (elastic#1…

…03440)

this removes smb test fixture to use test container instead of relying
on our gradle test fixture plugin.

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -134,7 +134,7 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
             }
             uploadFile.getParentFile().mkdirs();
             createBuildArchiveTar(parameters.getFilteredFiles().get(), parameters.getProjectDir().get(), uploadFile);
-            if (uploadFile.exists() && System.getenv("BUILDKITE").equals("true")) {
+            if (uploadFile.exists() && "true".equals(System.getenv("BUILDKITE"))) {
                 String uploadFilePath = "build/" + uploadFile.getName();
                 try {
                     System.out.println("Uploading buildkite artifact: " + uploadFilePath + "...");

test/fixtures/minio-fixture/build.gradle

@@ -11,26 +11,20 @@ apply plugin: 'elasticsearch.cache-test-fixtures'
 description = 'Fixture for MinIO Storage service'
 
 configurations.all {
-  transitive = false
+  exclude group: 'org.hamcrest', module: 'hamcrest-core'
 }
-
 dependencies {
-  testImplementation project(':test:framework')
-
+  testImplementation (project(':test:framework'))
   api "junit:junit:${versions.junit}"
   api project(':test:fixtures:testcontainer-utils')
-  api "org.testcontainers:testcontainers:${versions.testcontainer}"
-  implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
-  implementation "org.slf4j:slf4j-api:${versions.slf4j}"
-  implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}"
-
-  runtimeOnly "com.github.docker-java:docker-java-transport-zerodep:${versions.dockerJava}"
-  runtimeOnly "com.github.docker-java:docker-java-transport:${versions.dockerJava}"
-  runtimeOnly "com.github.docker-java:docker-java-core:${versions.dockerJava}"
-  runtimeOnly "org.apache.commons:commons-compress:${versions.commonsCompress}"
-  runtimeOnly "org.rnorth.duct-tape:duct-tape:${versions.ductTape}"
+  api("org.testcontainers:testcontainers:${versions.testcontainer}") {
+    transitive = false
+  }
+  api("com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"){
+    transitive = false
+  }
 
   // ensure we have proper logging during when used in tests
   runtimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}"
-  runtimeOnly "org.hamcrest:hamcrest:${versions.hamcrest}"
+  runtimeOnly("org.hamcrest:hamcrest:${versions.hamcrest}")
 }

test/fixtures/testcontainer-utils/build.gradle

@@ -1,6 +1,5 @@
 apply plugin: 'elasticsearch.java'
 
-
 configurations.all {
   transitive = false
 }
@@ -10,6 +9,13 @@ dependencies {
   api "junit:junit:${versions.junit}"
   api "org.testcontainers:testcontainers:${versions.testcontainer}"
   implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  api "com.github.docker-java:docker-java-api:${versions.dockerJava}"
   implementation "org.slf4j:slf4j-api:${versions.slf4j}"
-  implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}"
+  runtimeOnly "com.github.docker-java:docker-java-transport-zerodep:${versions.dockerJava}"
+  runtimeOnly "com.github.docker-java:docker-java-core:${versions.dockerJava}"
+  runtimeOnly "org.apache.commons:commons-compress:${versions.commonsCompress}"
+  runtimeOnly "org.rnorth.duct-tape:duct-tape:${versions.ductTape}"
+  runtimeOnly "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+  runtimeOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+
 }

test/fixtures/testcontainer-utils/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java

@@ -19,6 +19,6 @@ public class TestContainersThreadFilter implements ThreadFilter {
     public boolean reject(Thread t) {
         return t.getName().startsWith("testcontainers-")
             || t.getName().startsWith("ducttape")
-            || t.getName().startsWith("ForkJoinPool.commonPool-worker-1");
+            || t.getName().startsWith("ForkJoinPool.commonPool-worker-");
     }
 }

x-pack/qa/openldap-tests/build.gradle

@@ -5,11 +5,8 @@ dependencies {
   testImplementation(testArtifact(project(xpackModule('core'))))
   testImplementation project(":x-pack:test:idp-fixture")
   testImplementation "junit:junit:${versions.junit}"
-  testImplementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  testImplementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 }
 
-
 tasks.named('test') {
   // test suite uses jks which is not supported in fips mode
   systemProperty 'tests.security.manager', 'false'

x-pack/qa/third-party/active-directory/build.gradle

@@ -1,12 +1,15 @@
 apply plugin: 'elasticsearch.standalone-test'
-apply plugin: 'elasticsearch.test.fixtures'
 
+configurations.all {
+  exclude group: 'org.slf4j', module: 'slf4j-nop'
+}
 dependencies {
+  testImplementation project(':test:framework')
   testImplementation project(xpackModule('core'))
   testImplementation project(xpackModule('security'))
-  testImplementation(testArtifact(project(xpackModule('security'))))}
-
-testFixtures.useFixture ":x-pack:test:smb-fixture"
+  testImplementation(testArtifact(project(xpackModule('security'))))
+  testImplementation project(":x-pack:test:smb-fixture")
+}
 
 // add test resources from security, so tests can use example certs
 tasks.named("processTestResources").configure {
@@ -23,6 +26,7 @@ tasks.named("forbiddenPatterns").configure {
 }
 
 tasks.named("test").configure {
+  systemProperty 'tests.security.manager', 'false'
   include '**/*IT.class'
   include '**/*Tests.class'
 }

x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ADLdapUserSearchSessionFactoryTests.java

@@ -63,7 +63,7 @@ public void testUserSearchWithActiveDirectory() throws Exception {
         String groupSearchBase = "DC=ad,DC=test,DC=elasticsearch,DC=com";
         String userSearchBase = "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com";
         Settings settings = Settings.builder()
-            .put("url", ActiveDirectorySessionFactoryTests.AD_LDAP_URL)
+            .put("url", smbFixture.getAdLdapUrl())
             .put("group_search.base_dn", groupSearchBase)
             .put("user_search.base_dn", userSearchBase)
             .put("bind_dn", "[email protected]")

x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractActiveDirectoryTestCase.java

@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.security.authc.ldap;
 
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import com.unboundid.ldap.sdk.LDAPConnection;
 import com.unboundid.ldap.sdk.LDAPConnectionPool;
 import com.unboundid.ldap.sdk.LDAPException;
@@ -18,13 +19,16 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.fixtures.smb.SmbTestContainer;
+import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.ldap.ActiveDirectorySessionFactorySettings;
 import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope;
 import org.elasticsearch.xpack.core.security.authc.ldap.support.SessionFactorySettings;
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.junit.Before;
+import org.junit.ClassRule;
 
 import java.io.IOException;
 import java.nio.file.FileVisitResult;
@@ -39,23 +43,19 @@
 
 import static org.elasticsearch.xpack.core.security.authc.RealmSettings.getFullSettingKey;
 
+@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
 public abstract class AbstractActiveDirectoryTestCase extends ESTestCase {
 
+    @ClassRule
+    public static final SmbTestContainer smbFixture = new SmbTestContainer();
     // follow referrals defaults to false here which differs from the default value of the setting
     // this is needed to prevent test logs being filled by errors as the default configuration of
     // the tests run against a vagrant samba4 instance configured as a domain controller with the
     // ports mapped into the ephemeral port range and there is the possibility of incorrect results
     // as we cannot control the URL of the referral which may contain a non-resolvable DNS name as
     // this name would be served by the samba4 instance
     public static final Boolean FOLLOW_REFERRALS = Booleans.parseBoolean(getFromEnv("TESTS_AD_FOLLOW_REFERRALS", "false"));
-    public static final String AD_LDAP_URL = getFromEnv("TESTS_AD_LDAP_URL", "ldaps://localhost:" + getFromProperty("636"));
-    public static final String AD_LDAP_GC_URL = getFromEnv("TESTS_AD_LDAP_GC_URL", "ldaps://localhost:" + getFromProperty("3269"));
-    public static final String PASSWORD = getFromEnv("TESTS_AD_USER_PASSWORD", "Passw0rd");
-    public static final String AD_LDAP_PORT = getFromEnv("TESTS_AD_LDAP_PORT", getFromProperty("389"));
-
-    public static final String AD_LDAPS_PORT = getFromEnv("TESTS_AD_LDAPS_PORT", getFromProperty("636"));
-    public static final String AD_GC_LDAP_PORT = getFromEnv("TESTS_AD_GC_LDAP_PORT", getFromProperty("3268"));
-    public static final String AD_GC_LDAPS_PORT = getFromEnv("TESTS_AD_GC_LDAPS_PORT", getFromProperty("3269"));
+    public static final String PASSWORD = "Passw0rd";
     public static final String AD_DOMAIN = "ad.test.elasticsearch.com";
 
     protected SSLService sslService;
@@ -108,10 +108,6 @@ Settings buildAdSettings(
             .put(getFullSettingKey(realmId, ActiveDirectorySessionFactorySettings.AD_DOMAIN_NAME_SETTING), adDomainName)
             .put(getFullSettingKey(realmName, ActiveDirectorySessionFactorySettings.AD_USER_SEARCH_BASEDN_SETTING), userSearchDN)
             .put(getFullSettingKey(realmName, ActiveDirectorySessionFactorySettings.AD_USER_SEARCH_SCOPE_SETTING), scope)
-            .put(getFullSettingKey(realmName, ActiveDirectorySessionFactorySettings.AD_LDAP_PORT_SETTING), AD_LDAP_PORT)
-            .put(getFullSettingKey(realmName, ActiveDirectorySessionFactorySettings.AD_LDAPS_PORT_SETTING), AD_LDAPS_PORT)
-            .put(getFullSettingKey(realmName, ActiveDirectorySessionFactorySettings.AD_GC_LDAP_PORT_SETTING), AD_GC_LDAP_PORT)
-            .put(getFullSettingKey(realmName, ActiveDirectorySessionFactorySettings.AD_GC_LDAPS_PORT_SETTING), AD_GC_LDAPS_PORT)
             .put(getFullSettingKey(realmId, SessionFactorySettings.FOLLOW_REFERRALS_SETTING), FOLLOW_REFERRALS)
             .putList(getFullSettingKey(realmId, SSLConfigurationSettings.CAPATH_SETTING_REALM), certificatePaths);
         if (randomBoolean()) {
@@ -153,11 +149,4 @@ private static String getFromEnv(String envVar, String defaultValue) {
         final String value = System.getenv(envVar);
         return value == null ? defaultValue : value;
     }
-
-    private static String getFromProperty(String port) {
-        String key = "test.fixtures.smb-fixture.tcp." + port;
-        final String value = System.getProperty(key);
-        assertNotNull("Expected the actual value for port " + port + " to be in system property " + key, value);
-        return value;
-    }
 }

x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/AbstractAdLdapRealmTestCase.java

@@ -6,6 +6,8 @@
  */
 package org.elasticsearch.xpack.security.authc.ldap;
 
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
 import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchSecurityException;
@@ -21,18 +23,20 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.test.fixtures.smb.SmbTestContainer;
+import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequestBuilder;
 import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateAction;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequest;
 import org.elasticsearch.xpack.core.security.action.user.AuthenticateResponse;
-import org.elasticsearch.xpack.core.security.authc.ldap.ActiveDirectorySessionFactorySettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.ClassRule;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -47,14 +51,9 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.xpack.core.security.authc.RealmSettings.getFullSettingKey;
 import static org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope.ONE_LEVEL;
 import static org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope.SUB_TREE;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
-import static org.elasticsearch.xpack.security.authc.ldap.AbstractActiveDirectoryTestCase.AD_GC_LDAPS_PORT;
-import static org.elasticsearch.xpack.security.authc.ldap.AbstractActiveDirectoryTestCase.AD_GC_LDAP_PORT;
-import static org.elasticsearch.xpack.security.authc.ldap.AbstractActiveDirectoryTestCase.AD_LDAPS_PORT;
-import static org.elasticsearch.xpack.security.authc.ldap.AbstractActiveDirectoryTestCase.AD_LDAP_PORT;
 import static org.elasticsearch.xpack.security.test.SecurityTestUtils.writeFile;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -63,6 +62,7 @@
  * This test assumes all subclass tests will be of type SUITE.  It picks a random realm configuration for the tests, and
  * writes a group to role mapping file for each node.
  */
+@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
 public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase {
 
     public static final String XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL = "xpack.security.authc.realms.active_directory.external";
@@ -72,6 +72,9 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase
     public static final String PHILANTHROPISTS_INDEX = "philanthropists";
     public static final String SECURITY_INDEX = "security";
 
+    @ClassRule
+    public static final SmbTestContainer smbFixture = new SmbTestContainer();
+
     private static final RoleMappingEntry[] AD_ROLE_MAPPING = new RoleMappingEntry[] {
         new RoleMappingEntry("SHIELD:  [ \"CN=SHIELD,CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com\" ]", """
             {
@@ -359,12 +362,8 @@ enum RealmConfig {
                 .put(XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL + ".domain_name", ActiveDirectorySessionFactoryTests.AD_DOMAIN)
                 .put(XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL + ".group_search.base_dn", "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com")
                 .put(XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL + ".group_search.scope", randomBoolean() ? SUB_TREE : ONE_LEVEL)
-                .put(XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL + ".url", ActiveDirectorySessionFactoryTests.AD_LDAP_URL)
+                .put(XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL + ".url", smbFixture.getAdLdapUrl())
                 .put(XPACK_SECURITY_AUTHC_REALMS_AD_EXTERNAL + ".follow_referrals", ActiveDirectorySessionFactoryTests.FOLLOW_REFERRALS)
-                .put(getFullSettingKey("external", ActiveDirectorySessionFactorySettings.AD_LDAP_PORT_SETTING), AD_LDAP_PORT)
-                .put(getFullSettingKey("external", ActiveDirectorySessionFactorySettings.AD_LDAPS_PORT_SETTING), AD_LDAPS_PORT)
-                .put(getFullSettingKey("external", ActiveDirectorySessionFactorySettings.AD_GC_LDAP_PORT_SETTING), AD_GC_LDAP_PORT)
-                .put(getFullSettingKey("external", ActiveDirectorySessionFactorySettings.AD_GC_LDAPS_PORT_SETTING), AD_GC_LDAPS_PORT)
                 .build(),
             "active_directory"
         ),
@@ -373,7 +372,7 @@ enum RealmConfig {
             true,
             AD_ROLE_MAPPING,
             Settings.builder()
-                .put(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".url", ActiveDirectorySessionFactoryTests.AD_LDAP_URL)
+                .put(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".url", smbFixture.getAdLdapUrl())
                 .put(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".group_search.base_dn", "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com")
                 .put(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".group_search.scope", randomBoolean() ? SUB_TREE : ONE_LEVEL)
                 .putList(
@@ -389,7 +388,7 @@ enum RealmConfig {
             true,
             AD_ROLE_MAPPING,
             Settings.builder()
-                .put(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".url", ActiveDirectorySessionFactoryTests.AD_LDAP_URL)
+                .put(XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".url", smbFixture.getAdLdapUrl())
                 .putList(
                     XPACK_SECURITY_AUTHC_REALMS_LDAP_EXTERNAL + ".user_dn_templates",
                     "cn={0},CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com"

x-pack/qa/third-party/active-directory/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryGroupsResolverTests.java

@@ -6,15 +6,19 @@
  */
 package org.elasticsearch.xpack.security.authc.ldap;
 
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import com.unboundid.ldap.sdk.Filter;
 
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.test.fixtures.smb.SmbTestContainer;
+import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.ldap.support.LdapSearchScope;
 import org.elasticsearch.xpack.core.security.support.NoOpLogger;
 import org.junit.Before;
+import org.junit.ClassRule;
 
 import java.util.List;
 import java.util.regex.Pattern;
@@ -24,12 +28,16 @@
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 
+@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
 public class ActiveDirectoryGroupsResolverTests extends GroupsResolverTestCase {
 
     private static final String BRUCE_BANNER_DN = "cn=Bruce Banner,CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com";
 
     private static final RealmConfig.RealmIdentifier REALM_ID = new RealmConfig.RealmIdentifier("active_directory", "ad");
 
+    @ClassRule
+    public static final SmbTestContainer smbFixture = new SmbTestContainer();
+
     @Before
     public void setReferralFollowing() {
         ldapConnection.getConnectionOptions().setFollowReferrals(AbstractActiveDirectoryTestCase.FOLLOW_REFERRALS);
@@ -145,7 +153,7 @@ private void assertValidSidQuery(Filter query, String[] expectedSids) {
 
     @Override
     protected String ldapUrl() {
-        return ActiveDirectorySessionFactoryTests.AD_LDAP_URL;
+        return smbFixture.getAdLdapUrl();
     }
 
     @Override