Fix gradle config and add readme

NEW_TEST_FRAMEWORK.md

@@ -0,0 +1,39 @@
+# New test framework
+
+Decision was made that new tests should be fully independent of existing [(old)](src/test) tests. It means that the new tests must:
+* not use any of classes or resources located in [src/test/...](src/test),
+* not use any of already declared test dependencies,
+* be located in separate folder.
+
+Thanks to that we can keep the possibility to execute the old tests, incrementally add new tests, and replace the old tests with new ones in the future.
+
+
+## Project structure
+
+Tests are located in two directories: 
+* the source code of the existing tests is located in the [src/test/java](src/test/java) directory,
+* the [src/newTest/java](src/newTest/java) directory contains the source code of tests implemented using the new test framework.
+
+A new source set called `newTest` has been defined in [build.gradle](build.gradle), and configured:
+* the production classes from the main source set are added to the compile time classpath,
+* the production classes from the main source set are added to the runtime classpath,
+* the source directory of new tests is set to `src/newTest/java`,
+* the resource directory of new tests is set to `src/newTest/resources`.
+
+## Dependencies
+
+The Java Plugin automatically creates configurations for each of defined source sets.
+Names of configurations are prefixed by the name of the source set, e.g. **newTest**RuntimeOnly, **newTest**CompileOnly.
+`newTestImplementation` and `newTestRuntimeOnly` have been configured to extend from `implementation` and `runtimeOnly` respectively. 
+As a result all the declared dependencies of the production code also become dependencies of the new tests.
+
+## Running tests
+
+There is a new task called `newTest` that runs new tests. It has been configured to run after the `test` task, e.g. when 
+```
+./gradlew clean build
+```
+command is executed. In order to run just the new test, use command:
+```
+./gradle clean newTest
+```

build.gradle

@@ -84,6 +84,12 @@ forbiddenPatterns.enabled = false
 testingConventions.enabled = false
 // Conflicts between runtime kafka-clients:3.0.1 & testRuntime kafka-clients:3.0.1:test
 jarHell.enabled = false
+tasks.whenTaskAdded {task ->
+    if(task.name.contains("forbiddenApisNewTest")) {
+        task.enabled = false
+    }
+}
+
 
 test {
     include '**/*.class'
@@ -210,24 +216,57 @@ bundlePlugin {
     }
 }
 
-configurations.all {
-    resolutionStrategy {
-        force 'commons-codec:commons-codec:1.14'
-        force 'org.slf4j:slf4j-api:1.7.30'
-        force 'org.scala-lang:scala-library:2.13.8'
-        force 'commons-io:commons-io:2.11.0'
-        force "com.fasterxml.jackson:jackson-bom:${versions.jackson}"
-        force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-        force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}"
-        force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
-        force "io.netty:netty-buffer:${versions.netty}"
-        force "io.netty:netty-common:${versions.netty}"
-        force "io.netty:netty-handler:${versions.netty}"
-        force "io.netty:netty-transport:${versions.netty}"
-        force "io.netty:netty-transport-native-unix-common:${versions.netty}"
+configurations {
+    all {
+        resolutionStrategy {
+            force 'commons-codec:commons-codec:1.14'
+            force 'org.slf4j:slf4j-api:1.7.30'
+            force 'org.scala-lang:scala-library:2.13.8'
+            force 'commons-io:commons-io:2.11.0'
+            force "com.fasterxml.jackson:jackson-bom:${versions.jackson}"
+            force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+            force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}"
+            force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+            force "io.netty:netty-buffer:${versions.netty}"
+            force "io.netty:netty-common:${versions.netty}"
+            force "io.netty:netty-handler:${versions.netty}"
+            force "io.netty:netty-transport:${versions.netty}"
+        }
+    }
+
+    newTestImplementation.extendsFrom implementation
+    newTestRuntimeOnly.extendsFrom runtimeOnly
+}
+
+//create source set 'newTest'
+//add classes from the main source set to the compilation and runtime classpaths of the newTest
+sourceSets {
+    newTest {
+        java {
+            compileClasspath += sourceSets.main.output
+            runtimeClasspath += sourceSets.main.output
+            srcDir file ('src/newTest/java')
+        }
+        resources {
+            srcDir file('src/newTest/resources')
+        }
     }
 }
 
+//add new task that runs new tests
+task newTest(type: Test) {
+    description = 'Run new tests.'
+    group = 'verification'
+    testClassesDirs = sourceSets.newTest.output.classesDirs
+    classpath = sourceSets.newTest.runtimeClasspath
+
+    //run the newTest task after the test task
+    shouldRunAfter test
+}
+
+//run the newTest task before the check task
+check.dependsOn newTest
+
 dependencies {
     implementation 'jakarta.annotation:jakarta.annotation-api:1.3.5'
     implementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}"
@@ -350,6 +389,17 @@ dependencies {
     implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
 
     compileOnly "org.opensearch:opensearch:${opensearch_version}"
+
+    //new test framework:
+    newTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.7.1') {
+        exclude(group: 'junit', module: 'junit')
+    }
+    newTestImplementation 'junit:junit:4.13.2'
+    newTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
+    newTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
+    newTestImplementation 'commons-io:commons-io:2.7'
+    newTestImplementation 'org.apache.logging.log4j:log4j-core:2.17.1'
+    newTestImplementation 'org.hamcrest:hamcrest:2.1'
 }
 
 jar {

src/main/java/org/opensearch/security/support/PemKeyReader.java

@@ -274,15 +274,19 @@ public static X509Certificate[] loadCertificatesFromFile(String file) throws Exc
             return null;
         }
 
-        CertificateFactory fact = CertificateFactory.getInstance("X.509");
         try(FileInputStream is = new FileInputStream(file)) {
-            Collection<? extends Certificate> certs = fact.generateCertificates(is);
-            X509Certificate[] x509Certs = new X509Certificate[certs.size()];
-            int i=0;
-            for(Certificate cert: certs) {
-                x509Certs[i++] = (X509Certificate) cert;
-            }
-            return x509Certs;
+        	return loadCertificatesFromStream(is);
+        }
+
+    }
+
+    public static X509Certificate[] loadCertificatesFromFile(File file) throws Exception {
+        if(file == null) {
+            return null;
+        }
+
+        try(FileInputStream is = new FileInputStream(file)) {
+        	return loadCertificatesFromStream(is);
         }
 
     }

src/newTest/java/org/opensearch/node/PluginAwareNode.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2015-2018 _floragunn_ GmbH
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.node;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.opensearch.common.settings.Settings;
+import org.opensearch.plugins.Plugin;
+
+public class PluginAwareNode extends Node {
+
+    private final boolean clusterManagerEligible;
+
+    @SafeVarargs
+    public PluginAwareNode(boolean clusterManagerEligible, final Settings preparedSettings, final Class<? extends Plugin>... plugins) {
+    	super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, Collections.emptyMap(), null, () -> System.getenv("HOSTNAME")), Arrays.asList(plugins), true);
+        this.clusterManagerEligible = clusterManagerEligible;
+    }
+
+
+    public boolean isClusterManagerEligible() {
+        return clusterManagerEligible;
+    }
+}

src/newTest/java/org/opensearch/test/GenericIntegrationTest.java

@@ -0,0 +1,74 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.test;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import org.apache.http.HttpStatus;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.opensearch.test.framework.TestIndex;
+import org.opensearch.test.framework.TestSecurityConfig;
+import org.opensearch.test.framework.TestSecurityConfig.Role;
+import org.opensearch.test.framework.cluster.ClusterManager;
+import org.opensearch.test.framework.cluster.LocalCluster;
+import org.opensearch.test.framework.cluster.TestRestClient;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL;
+import static org.opensearch.test.framework.TestSecurityConfig.User.USER_ADMIN;
+
+/**
+ * WIP
+ * Generic test class that demonstrates how to use the test framework to 
+ * set up a test cluster with users, roles, indices and data, and how to
+ * implement tests. One main goal here is to make tests self-contained.
+ */
+@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class GenericIntegrationTest {
+
+    // define indices used in this test
+    private final static TestIndex INDEX_A = TestIndex.name("index-a").build();
+    private final static TestIndex INDEX_B = TestIndex.name("index-b").build();
+
+    private final static TestSecurityConfig.User INDEX_A_USER = new TestSecurityConfig.User("index_a_user")
+            .roles(new Role("index_a_role").indexPermissions("*").on(INDEX_A).clusterPermissions("*"));
+
+
+    // build our test cluster as a ClassRule
+    @ClassRule
+    public static LocalCluster cluster = new LocalCluster.Builder().clusterConfiguration(ClusterManager.THREE_MASTERS)
+    	.authc(AUTHC_HTTPBASIC_INTERNAL)
+    	.users(USER_ADMIN, INDEX_A_USER)
+    	.indices(INDEX_A, INDEX_B).build();
+
+    @Test
+    public void testAdminUserHasAccessToAllIndices() throws Exception {
+        try (TestRestClient client = cluster.getRestClient(USER_ADMIN)) {
+			assertThat(client.get("*/_search?pretty").getStatusCode(), equalTo(HttpStatus.SC_OK));
+        }        
+    }
+
+    @Test
+    public void testIndexAUserHasOnlyAccessToIndexA() throws Exception {
+        try (TestRestClient client = cluster.getRestClient(INDEX_A_USER)) {        	
+			assertThat(client.get("index-a/_search?pretty").getStatusCode(), equalTo(HttpStatus.SC_OK));            
+            // demo: work with JSON response body and check values
+			assertThat(client.get("index-a/_search?pretty").getIntFromJsonBody("/_source/hits/value"), equalTo(0));            
+			assertThat(client.get("index-b/_search?pretty").getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN));
+        }
+    }    
+
+}

src/newTest/java/org/opensearch/test/PrivilegesEvaluatorTest.java

@@ -0,0 +1,71 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.test;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import org.apache.http.HttpStatus;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.opensearch.test.framework.TestSecurityConfig;
+import org.opensearch.test.framework.TestSecurityConfig.Role;
+import org.opensearch.test.framework.cluster.ClusterManager;
+import org.opensearch.test.framework.cluster.LocalCluster;
+import org.opensearch.test.framework.cluster.TestRestClient;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL;
+
+/**
+ * This is a port for the test
+ * org.opensearch.security.privileges.PrivilegesEvaluatorTest to the new test
+ * framework for direct comparison
+ */
+@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class PrivilegesEvaluatorTest {
+
+	protected final static TestSecurityConfig.User NEGATIVE_LOOKAHEAD = new TestSecurityConfig.User(
+			"negative_lookahead_user")
+			.roles(new Role("negative_lookahead_role").indexPermissions("read").on("/^(?!t.*).*/")
+					.clusterPermissions("cluster_composite_ops"));
+
+	protected final static TestSecurityConfig.User NEGATED_REGEX = new TestSecurityConfig.User("negated_regex_user")
+			.roles(new Role("negated_regex_role").indexPermissions("read").on("/^[a-z].*/")
+					.clusterPermissions("cluster_composite_ops"));
+
+	@ClassRule
+	public static LocalCluster cluster = new LocalCluster.Builder()
+			.clusterConfiguration(ClusterManager.THREE_MASTERS).authc(AUTHC_HTTPBASIC_INTERNAL)
+			.users(NEGATIVE_LOOKAHEAD, NEGATED_REGEX).build();
+
+	@Test
+	public void testNegativeLookaheadPattern() throws Exception {
+
+		try (TestRestClient client = cluster.getRestClient(NEGATIVE_LOOKAHEAD)) {
+			assertThat(client.get("*/_search").getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN));
+			assertThat(client.get("r*/_search").getStatusCode(), equalTo(HttpStatus.SC_OK));
+		}
+	}
+
+	@Test
+	public void testRegexPattern() throws Exception {
+
+		try (TestRestClient client = cluster.getRestClient(NEGATED_REGEX)) {
+			assertThat(client.get("*/_search").getStatusCode(), equalTo(HttpStatus.SC_FORBIDDEN));
+			assertThat(client.get("r*/_search").getStatusCode(), equalTo(HttpStatus.SC_OK));
+		}
+
+	}
+}