From fff31d2b131bfc5d6bda22788ab4658cd2f101e7 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Wed, 28 Feb 2024 01:24:49 +0000 Subject: [PATCH 1/4] Add support for randomizing remote store enabled testing. Signed-off-by: Rishikesh1159 --- .../index/engine/MaxDocsLimitIT.java | 5 +++++ .../test/OpenSearchIntegTestCase.java | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/index/engine/MaxDocsLimitIT.java b/server/src/internalClusterTest/java/org/opensearch/index/engine/MaxDocsLimitIT.java index 8321630d34229..8146434c5cf6c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/engine/MaxDocsLimitIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/engine/MaxDocsLimitIT.java @@ -79,6 +79,11 @@ public Optional getEngineFactory(IndexSettings indexSettings) { } } + @Override + public boolean useRemoteBackedStorageRandomly() { + return true; + } + @Override protected boolean addMockInternalEngine() { return false; diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 47dd033834f1c..5d3ad34bbbb92 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -371,6 +371,10 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase { */ public static final String TESTS_CLUSTER_NAME = "tests.clustername"; + protected static final String REMOTE_BACKED_STORAGE_REPOSITORY_NAME = "test-remote-store-repo"; + + private Path remoteStoreRepositoryPath; + @BeforeClass public static void beforeClass() throws Exception { testClusterRule.beforeClass(); @@ -1896,6 +1900,17 @@ protected Settings nodeSettings(int nodeOrdinal) { logger.info("Randomly using Replication Strategy as {}.", replicationType.toString()); builder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), replicationType); } + + if (useRemoteBackedStorageRandomly()) { + boolean remoteBackedStorageEnabled = randomBoolean(); + logger.info("Remote Backed Storage (Remote Store) is set to {}.", remoteBackedStorageEnabled); + if (remoteBackedStorageEnabled) { + if (remoteStoreRepositoryPath == null) { + remoteStoreRepositoryPath = randomRepoPath().toAbsolutePath(); + } + builder.put(remoteStoreClusterSettings(REMOTE_BACKED_STORAGE_REPOSITORY_NAME, remoteStoreRepositoryPath)); + } + } return builder.build(); } @@ -1908,6 +1923,10 @@ protected boolean useRandomReplicationStrategy() { return false; } + protected boolean useRemoteBackedStorageRandomly() { + return false; + } + protected Path nodeConfigPath(int nodeOrdinal) { return null; } From b3f3abdd394c1f2d8c0be68625b3dbaf3d77fdd6 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Tue, 19 Mar 2024 12:32:47 +0000 Subject: [PATCH 2/4] fix spotless check Signed-off-by: Rishikesh1159 --- .../main/java/org/opensearch/test/OpenSearchIntegTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 14da574274ce5..51480b7c458e2 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -1912,7 +1912,7 @@ protected Settings nodeSettings(int nodeOrdinal) { if (remoteStoreRepositoryPath == null) { remoteStoreRepositoryPath = randomRepoPath().toAbsolutePath(); } - builder.put(remoteStoreClusterSettings(REMOTE_BACKED_STORAGE_REPOSITORY_NAME, remoteStoreRepositoryPath)); + builder.put(remoteStoreClusterSettings(REMOTE_BACKED_STORAGE_REPOSITORY_NAME, remoteStoreRepositoryPath)); } } return builder.build(); From b3346bd19e3b9ebe535d84d0cdc4311291526e98 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Thu, 21 Mar 2024 18:25:17 +0000 Subject: [PATCH 3/4] Updating logic to randomly use remote store once per cluster instead of once per node. Signed-off-by: Rishikesh1159 --- .../index/engine/MaxDocsLimitIT.java | 5 ----- .../recovery/ReplicaToPrimaryPromotionIT.java | 7 ++++++- .../test/OpenSearchIntegTestCase.java | 21 +++++++++++++------ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/index/engine/MaxDocsLimitIT.java b/server/src/internalClusterTest/java/org/opensearch/index/engine/MaxDocsLimitIT.java index 8146434c5cf6c..8321630d34229 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/engine/MaxDocsLimitIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/engine/MaxDocsLimitIT.java @@ -79,11 +79,6 @@ public Optional getEngineFactory(IndexSettings indexSettings) { } } - @Override - public boolean useRemoteBackedStorageRandomly() { - return true; - } - @Override protected boolean addMockInternalEngine() { return false; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java index 3df4ecff5250c..036ba247576bd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java @@ -56,6 +56,11 @@ protected int numberOfReplicas() { return 1; } + @Override + public boolean useRemoteBackedStorageRandomly() { + return true; + } + public void testPromoteReplicaToPrimary() throws Exception { final String indexName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); createIndex(indexName); @@ -65,7 +70,7 @@ public void testPromoteReplicaToPrimary() throws Exception { try (BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), numOfDocs)) { waitForDocs(numOfDocs, indexer); } - refresh(indexName); + refreshAndWaitForReplication(indexName); } assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 51480b7c458e2..151c5cabe0a5b 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -381,6 +381,10 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase { private Path remoteStoreRepositoryPath; + private ReplicationType randomReplicationType; + + private String randomStorageType; + @BeforeClass public static void beforeClass() throws Exception { testClusterRule.beforeClass(); @@ -1900,15 +1904,14 @@ protected Settings nodeSettings(int nodeOrdinal) { // Randomly set a replication strategy for the node. Replication Strategy can still be manually overridden by subclass if needed. if (useRandomReplicationStrategy()) { - ReplicationType replicationType = randomBoolean() ? ReplicationType.DOCUMENT : ReplicationType.SEGMENT; - logger.info("Randomly using Replication Strategy as {}.", replicationType.toString()); - builder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), replicationType); + logger.info("Randomly using Replication Strategy as {}.", randomReplicationType.toString()); + builder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), randomReplicationType); } + // Randomly set storage type for the node. Storage Type can still be manually overridden by subclass if needed. if (useRemoteBackedStorageRandomly()) { - boolean remoteBackedStorageEnabled = randomBoolean(); - logger.info("Remote Backed Storage (Remote Store) is set to {}.", remoteBackedStorageEnabled); - if (remoteBackedStorageEnabled) { + logger.info("Randomly using Storage Type as {}.", randomStorageType); + if (randomStorageType.equals("REMOTE_STORE")) { if (remoteStoreRepositoryPath == null) { remoteStoreRepositoryPath = randomRepoPath().toAbsolutePath(); } @@ -1970,6 +1973,12 @@ protected boolean ignoreExternalCluster() { } protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException { + if (useRandomReplicationStrategy()) { + randomReplicationType = randomBoolean() ? ReplicationType.DOCUMENT : ReplicationType.SEGMENT; + } + if (useRemoteBackedStorageRandomly()) { + randomStorageType = randomBoolean() ? "REMOTE_STORE" : "LOCAL"; + } String clusterAddresses = System.getProperty(TESTS_CLUSTER); if (Strings.hasLength(clusterAddresses) && ignoreExternalCluster() == false) { if (scope == Scope.TEST) { From dd0c698bc7a2ff62ee0b105e0d204da362a71ce0 Mon Sep 17 00:00:00 2001 From: Rishikesh1159 Date: Thu, 4 Apr 2024 06:53:47 +0000 Subject: [PATCH 4/4] Add Remote Store Randomization within Replication Type Randomization. Signed-off-by: Rishikesh1159 --- .../indices/IndicesRequestCacheIT.java | 1 + .../recovery/ReplicaToPrimaryPromotionIT.java | 2 +- .../test/OpenSearchIntegTestCase.java | 28 ++++++++----------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java index 52b4dad553180..ec5637cec6485 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java @@ -650,6 +650,7 @@ public void testCacheWithInvalidation() throws Exception { .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.refresh_interval", -1) ) .get() ); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java index 036ba247576bd..a2543f0592145 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java @@ -57,7 +57,7 @@ protected int numberOfReplicas() { } @Override - public boolean useRemoteBackedStorageRandomly() { + public boolean useRandomReplicationStrategy() { return true; } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 151c5cabe0a5b..a5d74df33f9d7 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -1902,20 +1902,18 @@ protected Settings nodeSettings(int nodeOrdinal) { builder.put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true); } - // Randomly set a replication strategy for the node. Replication Strategy can still be manually overridden by subclass if needed. + // Randomly set a Replication Strategy and storage type for the node. Both Replication Strategy and Storage Type can still be + // manually overridden by subclass if needed. if (useRandomReplicationStrategy()) { - logger.info("Randomly using Replication Strategy as {}.", randomReplicationType.toString()); - builder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), randomReplicationType); - } - - // Randomly set storage type for the node. Storage Type can still be manually overridden by subclass if needed. - if (useRemoteBackedStorageRandomly()) { - logger.info("Randomly using Storage Type as {}.", randomStorageType); - if (randomStorageType.equals("REMOTE_STORE")) { + if (randomReplicationType.equals(ReplicationType.SEGMENT) && randomStorageType.equals("REMOTE_STORE")) { + logger.info("Randomly using Replication Strategy as {} and Storage Type as {}.", randomReplicationType, randomStorageType); if (remoteStoreRepositoryPath == null) { remoteStoreRepositoryPath = randomRepoPath().toAbsolutePath(); } builder.put(remoteStoreClusterSettings(REMOTE_BACKED_STORAGE_REPOSITORY_NAME, remoteStoreRepositoryPath)); + } else { + logger.info("Randomly using Replication Strategy as {} and Storage Type as {}.", randomReplicationType, randomStorageType); + builder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), randomReplicationType); } } return builder.build(); @@ -1930,10 +1928,6 @@ protected boolean useRandomReplicationStrategy() { return false; } - protected boolean useRemoteBackedStorageRandomly() { - return false; - } - protected Path nodeConfigPath(int nodeOrdinal) { return null; } @@ -1975,9 +1969,11 @@ protected boolean ignoreExternalCluster() { protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException { if (useRandomReplicationStrategy()) { randomReplicationType = randomBoolean() ? ReplicationType.DOCUMENT : ReplicationType.SEGMENT; - } - if (useRemoteBackedStorageRandomly()) { - randomStorageType = randomBoolean() ? "REMOTE_STORE" : "LOCAL"; + if (randomReplicationType.equals(ReplicationType.SEGMENT)) { + randomStorageType = randomBoolean() ? "REMOTE_STORE" : "LOCAL"; + } else { + randomStorageType = "LOCAL"; + } } String clusterAddresses = System.getProperty(TESTS_CLUSTER); if (Strings.hasLength(clusterAddresses) && ignoreExternalCluster() == false) {