Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a test to ensure Index Metadata is incremented if system flag flipped to true #37

Draft
wants to merge 7 commits into
base: main
Choose a base branch
from
Draft

Add a test to ensure Index Metadata is incremented if system flag flipped to true #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8b6bbf1
WIP to stop and start local cluster
cwperks Nov 13, 2024
d3808f3
Get stop and start to work
cwperks Nov 13, 2024
d1759be
Cleanup test
cwperks Nov 13, 2024
81c8f42
Assert that index metadata version is incremented
cwperks Nov 13, 2024
dc025c7
Pass waitFor
cwperks Nov 13, 2024
9c5675e
Restart random node
cwperks Nov 13, 2024
ceede4a
Restart random data node and install plugin
cwperks Nov 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/integrationTest/java/org/opensearch/security/SystemIndexTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import org.junit.runner.RunWith;

import org.opensearch.core.rest.RestStatus;
import org.opensearch.security.http.ExampleSystemIndexPlugin;
import org.opensearch.security.plugin.SystemIndexPlugin1;
import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain;
import org.opensearch.test.framework.cluster.ClusterManager;
import org.opensearch.test.framework.cluster.LocalCluster;
Expand All @@ -44,7 +44,7 @@ public class SystemIndexTests {
.anonymousAuth(false)
.authc(AUTHC_DOMAIN)
.users(USER_ADMIN)
.plugin(ExampleSystemIndexPlugin.class)
.plugin(SystemIndexPlugin1.class)
.nodeSettings(
Map.of(
SECURITY_RESTAPI_ROLES_ENABLED,
Expand Down
139 changes: 139 additions & 0 deletions src/integrationTest/java/org/opensearch/security/SystemIndexUpgradeTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
*/
package org.opensearch.security;

import java.util.List;
import java.util.Map;

import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import org.awaitility.Awaitility;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;

import org.opensearch.core.rest.RestStatus;
import org.opensearch.security.plugin.SystemIndexPlugin1;
import org.opensearch.test.framework.TestSecurityConfig.AuthcDomain;
import org.opensearch.test.framework.cluster.ClusterManager;
import org.opensearch.test.framework.cluster.LocalCluster;
import org.opensearch.test.framework.cluster.LocalOpenSearchCluster;
import org.opensearch.test.framework.cluster.TestRestClient;
import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED;
import static org.opensearch.security.support.ConfigConstants.SECURITY_SYSTEM_INDICES_ENABLED_KEY;
import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS;
import static org.opensearch.test.framework.TestSecurityConfig.User.USER_ADMIN;

@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
public class SystemIndexUpgradeTests {

public static final AuthcDomain AUTHC_DOMAIN = new AuthcDomain("basic", 0).httpAuthenticatorWithChallenge("basic").backend("internal");

@ClassRule
public static final LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.DEFAULT)
.anonymousAuth(false)
.authc(AUTHC_DOMAIN)
.users(USER_ADMIN)
.loadConfigurationIntoIndex(true)
.nodeSettings(
Map.of(
SECURITY_RESTAPI_ROLES_ENABLED,
List.of("user_" + USER_ADMIN.getName() + "__" + ALL_ACCESS.getName()),
SECURITY_SYSTEM_INDICES_ENABLED_KEY,
true,
"node.max_local_storage_nodes",
2
)
)
.build();

@Before
public void setup() {
try (TestRestClient client = cluster.getRestClient(cluster.getAdminCertificate())) {
client.delete(SystemIndexPlugin1.SYSTEM_INDEX_1);
}
}

@Test
public void systemIndexShouldBeMarkedTrueInClusterState() throws InterruptedException {
int previousVersion;
System.out.println("cluster.nodes: " + cluster.nodes());
for (LocalOpenSearchCluster.Node node : cluster.nodes()) {
try (TestRestClient client = node.getRestClient(USER_ADMIN)) {
client.put(SystemIndexPlugin1.SYSTEM_INDEX_1);
HttpResponse response = client.get("_cluster/state/metadata/" + SystemIndexPlugin1.SYSTEM_INDEX_1);

assertThat(response.getStatusCode(), equalTo(RestStatus.OK.getStatus()));

boolean isSystem = response.bodyAsJsonNode()
.get("metadata")
.get("indices")
.get(SystemIndexPlugin1.SYSTEM_INDEX_1)
.get("system")
.asBoolean();
previousVersion = response.bodyAsJsonNode()
.get("metadata")
.get("indices")
.get(SystemIndexPlugin1.SYSTEM_INDEX_1)
.get("version")
.asInt();

System.out.println("node: " + node.getNodeName());
System.out.println("response.body: " + response.getBody());
System.out.println("isSystem: " + isSystem);
System.out.println("previousVersion: " + previousVersion);

assertThat(isSystem, equalTo(false));
}
}

cluster.addPlugin(SystemIndexPlugin1.class);

cluster.restartRandomNode();

for (LocalOpenSearchCluster.Node node : cluster.nodes()) {
try (TestRestClient client = node.getRestClient(USER_ADMIN)) {
System.out.println("node: " + node.getNodeName());
Awaitility.await().alias("Load default configuration").until(() -> client.getAuthInfo().getStatusCode(), equalTo(200));
HttpResponse catPluginsResponse = client.get("_cat/plugins");

System.out.println("catPluginsResponse: " + catPluginsResponse.getBody());
HttpResponse response = client.get("_cluster/state/metadata/" + SystemIndexPlugin1.SYSTEM_INDEX_1);

assertThat(response.getStatusCode(), equalTo(RestStatus.OK.getStatus()));

boolean isSystem = response.bodyAsJsonNode()
.get("metadata")
.get("indices")
.get(SystemIndexPlugin1.SYSTEM_INDEX_1)
.get("system")
.asBoolean();
int version = response.bodyAsJsonNode()
.get("metadata")
.get("indices")
.get(SystemIndexPlugin1.SYSTEM_INDEX_1)
.get("version")
.asInt();

System.out.println("response.body: " + response.getBody());
System.out.println("isSystem: " + isSystem);
System.out.println("version: " + version);

// assertThat(isSystem, equalTo(true));
// assertThat(version, greaterThan(previousVersion));
}
}
}
}
9 changes: 6 additions & 3 deletions ...curity/http/ExampleSystemIndexPlugin.java → ...h/security/plugin/SystemIndexPlugin1.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,24 @@
* compatible open source license.
*
*/
package org.opensearch.security.http;

package org.opensearch.security.plugin;

import java.util.Collection;
import java.util.Collections;

import org.opensearch.common.settings.Settings;
import org.opensearch.indices.SystemIndexDescriptor;
import org.opensearch.plugins.IdentityAwarePlugin;
import org.opensearch.plugins.Plugin;
import org.opensearch.plugins.SystemIndexPlugin;

public class ExampleSystemIndexPlugin extends Plugin implements SystemIndexPlugin {
public class SystemIndexPlugin1 extends Plugin implements SystemIndexPlugin, IdentityAwarePlugin {
public static final String SYSTEM_INDEX_1 = ".system-index1";

@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
final SystemIndexDescriptor systemIndexDescriptor = new SystemIndexDescriptor(".system-index1", "System index 1");
final SystemIndexDescriptor systemIndexDescriptor = new SystemIndexDescriptor(SYSTEM_INDEX_1, "System index 1");
return Collections.singletonList(systemIndexDescriptor);
}
}
49 changes: 47 additions & 2 deletions src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.junit.rules.ExternalResource;

import org.opensearch.client.Client;
import org.opensearch.cluster.health.ClusterHealthStatus;
import org.opensearch.common.settings.Settings;
import org.opensearch.node.PluginAwareNode;
import org.opensearch.plugins.Plugin;
Expand Down Expand Up @@ -138,6 +139,10 @@ public String getSnapshotDirPath() {
return localOpenSearchCluster.getSnapshotDirPath();
}

public void addPlugin(Class<? extends Plugin> plugin) {
this.plugins.add(plugin);
}

@Override
public void before() {
if (localOpenSearchCluster == null) {
Expand All @@ -163,6 +168,29 @@ protected void after() {
close();
}

public void stop() {
if (localOpenSearchCluster != null && localOpenSearchCluster.isStarted()) {
try {
localOpenSearchCluster.stop();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

public void restartRandomNode() {
if (localOpenSearchCluster != null && localOpenSearchCluster.isStarted()) {
try {
localOpenSearchCluster.restartRandomNode();
if (loadConfigurationIntoIndex) {
loadFromSecurityIndex();
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}

@Override
public void close() {
System.clearProperty(INIT_CONFIGURATION_DIR);
Expand Down Expand Up @@ -235,13 +263,18 @@ public Random getRandom() {
return localOpenSearchCluster.getRandom();
}

private void start() {
public void start() {
try {
NodeSettingsSupplier nodeSettingsSupplier = minimumOpenSearchSettingsSupplierFactory.minimumOpenSearchSettings(
sslOnly,
nodeSpecificOverride,
nodeOverride
);
if (localOpenSearchCluster != null) {
localOpenSearchCluster.plugins(plugins);
localOpenSearchCluster.start(ClusterHealthStatus.YELLOW);
return;
}
localOpenSearchCluster = new LocalOpenSearchCluster(
clusterName,
clusterManager,
Expand All @@ -251,7 +284,7 @@ private void start() {
expectedNodeStartupCount
);

localOpenSearchCluster.start();
localOpenSearchCluster.start(ClusterHealthStatus.GREEN);

if (loadConfigurationIntoIndex) {
initSecurityIndex(testSecurityConfig);
Expand Down Expand Up @@ -282,6 +315,18 @@ private void initSecurityIndex(TestSecurityConfig testSecurityConfig) {
}
}

private void loadFromSecurityIndex() {
log.info("Loading from OpenSearch Security index");
try (
Client client = new ContextHeaderDecoratorClient(
this.getInternalNodeClient(),
Map.of(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true")
)
) {
triggerConfigurationReload(client);
}
}

public void updateUserConfiguration(List<TestSecurityConfig.User> users) {
try (
Client client = new ContextHeaderDecoratorClient(
Expand Down
55 changes: 50 additions & 5 deletions src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ public class LocalOpenSearchCluster {
private final String clusterName;
private final ClusterManager clusterManager;
private final NodeSettingsSupplier nodeSettingsSupplier;
private final List<Class<? extends Plugin>> additionalPlugins;
private List<Class<? extends Plugin>> additionalPlugins;
private final List<Node> nodes = new ArrayList<>();
private final TestCertificates testCertificates;
private final Integer expectedNodeStartupCount;
Expand Down Expand Up @@ -129,6 +129,10 @@ public LocalOpenSearchCluster(
}
}

public void plugins(List<Class<? extends Plugin>> plugins) {
this.additionalPlugins = plugins;
}

public String getSnapshotDirPath() {
return snapshotDir.getAbsolutePath();
}
Expand All @@ -148,7 +152,7 @@ private long countNodesByType(NodeType nodeType) {
return getNodesByType(nodeType).stream().count();
}

public void start() throws Exception {
public void start(ClusterHealthStatus waitForStatus) throws Exception {
log.info("Starting {}", clusterName);

int clusterManagerNodeCount = clusterManager.getClusterManagerNodes();
Expand Down Expand Up @@ -199,14 +203,14 @@ public void start() throws Exception {
return;
}

log.info("Startup finished. Waiting for GREEN");
log.info("Startup finished. Waiting for " + waitForStatus);

int expectedCount = nodes.size();
if (expectedNodeStartupCount != null) {
expectedCount = expectedNodeStartupCount;
}

waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), expectedCount);
waitForCluster(waitForStatus, TimeValue.timeValueSeconds(10), expectedCount);
log.info("Started: {}", this);

}
Expand All @@ -219,12 +223,53 @@ public boolean isStarted() {
return started;
}

public void restartRandomNode() throws IOException {
List<CompletableFuture<Boolean>> stopFutures = new ArrayList<>();
List<Node> dataNodes = nodes.stream().filter(n -> DATA.equals(n.nodeType)).toList();
Node node = dataNodes.get(random.nextInt(dataNodes.size()));
stopFutures.add(node.stop(2, TimeUnit.SECONDS));
CompletableFuture.allOf(stopFutures.toArray(CompletableFuture[]::new)).join();
boolean allNodesStopped = stopFutures.stream().map(CompletableFuture::join).allMatch(result -> (Boolean.TRUE == result));

if (!allNodesStopped) {
throw new RuntimeException("Failed to stop single node in the cluster");
}

nodes.remove(node);

final var nodeSettings = clusterManager.getNonClusterManagerNodeSettings().get(0);

List<CompletableFuture<StartStage>> futures = new ArrayList<>();

System.out.println("nodes before: " + nodes);

Node replacementNode = new Node(node.nodeNumber, nodeSettings, node.transportPort, node.httpPort);

futures.add(replacementNode.start());

CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));

System.out.println("nodes after: " + nodes);

waitForCluster(ClusterHealthStatus.YELLOW, TimeValue.timeValueSeconds(10), nodes.size());

System.out.println("nodes after cluster health: " + nodes);
log.info("Started: {}", this);
}

public void stop() {
List<CompletableFuture<Boolean>> stopFutures = new ArrayList<>();
for (Node node : nodes) {
stopFutures.add(node.stop(2, TimeUnit.SECONDS));
}
CompletableFuture.allOf(stopFutures.toArray(CompletableFuture[]::new)).join();
boolean allNodesStopped = stopFutures.stream().map(CompletableFuture::join).allMatch(result -> (Boolean.TRUE == result));

if (!allNodesStopped) {
throw new RuntimeException("Failed to stop all nodes in the cluster");
}

nodes.clear();
}

public void destroy() {
Expand Down Expand Up @@ -280,7 +325,7 @@ private void retry() throws Exception {
this.seedHosts = null;
this.initialClusterManagerHosts = null;
createClusterDirectory("local_cluster_" + clusterName + "_retry_" + retry);
start();
start(ClusterHealthStatus.GREEN);
}

@SafeVarargs
Expand Down
Loading