From 45327709fc913f1e3c3c389c85c5f40da05e471b Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Wed, 5 Jul 2023 15:11:32 +0530 Subject: [PATCH 01/17] WIP - debug restore failures with replicas Signed-off-by: Bhumika Saini --- .../opensearch/remotestore/RemoteStoreIT.java | 244 +++++++++++++++--- 1 file changed, 215 insertions(+), 29 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index f01e4969b1fe7..b69bdf9660ffa 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -15,10 +15,12 @@ import org.opensearch.action.admin.indices.recovery.RecoveryResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; import org.opensearch.index.shard.RemoteStoreRefreshListener; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.plugins.Plugin; @@ -45,6 +47,9 @@ public class RemoteStoreIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; + private static final String INDEX_NAME_WILDCARD = "remote-store-test-*"; + private static final String INDEX_NAMES = "remote-store-test-idx-1,remote-store-test-idx-2,remote-store-test-index-1,remote-store-test-index-2"; + private static final String INDEX_NAMES_WILDCARD = "remote-store-test-idx-*,remote-store-test-index-*"; private static final String TOTAL_OPERATIONS = "total-operations"; private static final String REFRESHED_OR_FLUSHED_OPERATIONS = "refreshed-or-flushed-operations"; private static final String MAX_SEQ_NO_TOTAL = "max-seq-no-total"; @@ -72,16 +77,16 @@ private IndexResponse indexSingleDoc() { .get(); } - private Map indexData(int numberOfIterations, boolean invokeFlush) { + private Map indexData(int numberOfIterations, boolean invokeFlush, String index) { long totalOperations = 0; long refreshedOrFlushedOperations = 0; long maxSeqNo = -1; long maxSeqNoRefreshedOrFlushed = -1; for (int i = 0; i < numberOfIterations; i++) { if (invokeFlush) { - flush(INDEX_NAME); + flush(index); } else { - refresh(INDEX_NAME); + refresh(index); } maxSeqNoRefreshedOrFlushed = maxSeqNo; refreshedOrFlushedOperations = totalOperations; @@ -100,76 +105,257 @@ private Map indexData(int numberOfIterations, boolean invokeFlush) return indexingStats; } - private void verifyRestoredData(Map indexStats, boolean checkTotal) { + private void verifyRestoredData(Map indexStats, boolean checkTotal, String indexName) { String statsGranularity = checkTotal ? TOTAL_OPERATIONS : REFRESHED_OR_FLUSHED_OPERATIONS; String maxSeqNoGranularity = checkTotal ? MAX_SEQ_NO_TOTAL : MAX_SEQ_NO_REFRESHED_OR_FLUSHED; - ensureYellowAndNoInitializingShards(INDEX_NAME); - ensureGreen(INDEX_NAME); - assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), indexStats.get(statsGranularity)); + ensureYellowAndNoInitializingShards(indexName); + ensureGreen(indexName); + assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity)); IndexResponse response = indexSingleDoc(); assertEquals(indexStats.get(maxSeqNoGranularity) + 1, response.getSeqNo()); - refresh(INDEX_NAME); - assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), indexStats.get(statsGranularity) + 1); + refresh(indexName); + assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity) + 1); } - private void testRestoreFlow(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws IOException { - internalCluster().startDataOnlyNodes(3); - if (remoteTranslog) { - createIndex(INDEX_NAME, remoteTranslogIndexSettings(0)); - } else { - createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); + private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, boolean remoteTranslogEnabled, String indices, int replicaCount) { + internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); + internalCluster().startDataOnlyNodes(numDataOnlyNodes); + for (String index : indices.split(",")) { + if (remoteTranslogEnabled) { + createIndex(index, remoteTranslogIndexSettings(replicaCount)); + } else { + createIndex(index, remoteStoreIndexSettings(replicaCount)); + } + + ensureYellowAndNoInitializingShards(index); + ensureGreen(index); } - ensureYellowAndNoInitializingShards(INDEX_NAME); - ensureGreen(INDEX_NAME); + } + + /** + * Helper function to test restoring an index with no replication from remote store. Only primary node is dropped. + * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store. + * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. + * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. + * @throws IOException IO Exception. + */ + private void testRestoreFlow(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws IOException { + prepareCluster(0, 3, remoteTranslog, INDEX_NAME, 0); - Map indexStats = indexData(numberOfIterations, invokeFlush); + Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); - assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); + ensureGreen(INDEX_NAME); + verifyRestoredData(indexStats, remoteTranslog, INDEX_NAME); + } - if (remoteTranslog) { - verifyRestoredData(indexStats, true); - } else { - verifyRestoredData(indexStats, false); - } + /** + * Helper function to test restoring an index having replicas from remote store when all the nodes housing the primary/replica drop. + * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store. + * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. + * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. + * @throws IOException IO Exception. + */ + private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws IOException, InterruptedException { + prepareCluster(1, 2, remoteTranslog, INDEX_NAME, 1); + + Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(INDEX_NAME))); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); + ensureRed(INDEX_NAME); + + internalCluster().startDataOnlyNodes(2); + Thread.sleep(10000); + + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); + client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); + + ensureYellow(INDEX_NAME); + verifyRestoredData(indexStats, remoteTranslog, INDEX_NAME); } +// /** +// * Helper function to test restoring multiple indices from remote store when all the nodes housing the primary/replica drop. +// * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store. +// * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. +// * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. +// * @throws IOException IO Exception. +// */ +// private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws IOException { +// internalCluster().startDataOnlyNodes(3); +// if (remoteTranslog) { +// createIndex(INDEX_NAME, remoteTranslogIndexSettings(1)); +// } else { +// createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); +// } +// ensureYellowAndNoInitializingShards(INDEX_NAME); +// ensureGreen(INDEX_NAME); +// +// Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); +// +// internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); +// internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(INDEX_NAME))); +// assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); +// +// client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); +// ensureGreen(INDEX_NAME); +// +// if (remoteTranslog) { +// verifyRestoredData(indexStats, true, INDEX_NAME); +// } else { +// verifyRestoredData(indexStats, false, INDEX_NAME); +// } +// } + + /** + * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. + * @throws IOException IO Exception. + */ public void testRemoteSegmentStoreRestoreWithNoDataPostCommit() throws IOException { testRestoreFlow(false, 1, true); } + /** + * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. + * @throws IOException IO Exception. + */ public void testRemoteSegmentStoreRestoreWithNoDataPostRefresh() throws IOException { testRestoreFlow(false, 1, false); } + /** + * Simulates data restored until the refreshed data in Remote Segment Store + * and data loss for the unrefreshed data. + * @throws IOException IO Exception. + */ public void testRemoteSegmentStoreRestoreWithRefreshedData() throws IOException { testRestoreFlow(false, randomIntBetween(2, 5), false); } + /** + * Simulates data restored until the refreshed data in Remote Segment Store + * and data loss for the unrefreshed data. + * @throws IOException IO Exception. + */ public void testRemoteSegmentStoreRestoreWithCommittedData() throws IOException { testRestoreFlow(false, randomIntBetween(2, 5), true); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") +// @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") + /** + * Simulates all data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ public void testRemoteTranslogRestoreWithNoDataPostCommit() throws IOException { testRestoreFlow(true, 1, true); } + /** + * Simulates all data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ public void testRemoteTranslogRestoreWithNoDataPostRefresh() throws IOException { testRestoreFlow(true, 1, false); } + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ public void testRemoteTranslogRestoreWithRefreshedData() throws IOException { testRestoreFlow(true, randomIntBetween(2, 5), false); } + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ public void testRemoteTranslogRestoreWithCommittedData() throws IOException { testRestoreFlow(true, randomIntBetween(2, 5), true); } +// /** +// * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. +// * @throws IOException IO Exception. +// */ +// public void testRSSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOException { +// testRestoreFlowBothPrimaryReplicasDown(false, 1, true); +// } +// +// /** +// * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. +// * @throws IOException IO Exception. +// */ +// public void testRSSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { +// testRestoreFlowBothPrimaryReplicasDown(false, 1, false); +// } + + /** + * Simulates data restored until the refreshed data in Remote Segment Store + * and data loss for the unrefreshed data. + * @throws IOException IO Exception. + */ + public void testRSSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOException, InterruptedException { + testRestoreFlowBothPrimaryReplicasDown(false, randomIntBetween(2, 5), false); + } + +// /** +// * Simulates data restored until the refreshed data in Remote Segment Store +// * and data loss for the unrefreshed data. +// * @throws IOException IO Exception. +// */ +// public void testRSSRestoreWithCommittedDataPrimaryReplicaDown() throws IOException { +// testRestoreFlowBothPrimaryReplicasDown(false, randomIntBetween(2, 5), true); +// } +// +//// @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") +// /** +// * Simulates all data restored using Remote Translog Store. +// * @throws IOException IO Exception. +// */ +// public void testRTSWithNoDataPostCommitPrimaryReplicaDown() throws IOException { +// testRestoreFlowBothPrimaryReplicasDown(true, 1, true); +// } +// +// /** +// * Simulates all data restored using Remote Translog Store. +// * @throws IOException IO Exception. +// */ +// public void testRTSWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { +// testRestoreFlowBothPrimaryReplicasDown(true, 1, false); +// } +// +// /** +// * Simulates refreshed data restored using Remote Segment Store +// * and unrefreshed data restored using Remote Translog Store. +// * @throws IOException IO Exception. +// */ +// public void testRTSWithRefreshedDataPrimaryReplicaDown() throws IOException { +// testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), false); +// } +// +// /** +// * Simulates refreshed data restored using Remote Segment Store +// * and unrefreshed data restored using Remote Translog Store. +// * @throws IOException IO Exception. +// */ +// public void testRTSWithCommittedDataPrimaryReplicaDown() throws IOException { +// testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), true); +// } + + + +// public void testRemoteTranslogRestoreWithCommittedDataIndexPattern() throws IOException { +// testRestoreFlow(true, randomIntBetween(2, 5), true); +// } + private void testPeerRecovery(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws Exception { internalCluster().startDataOnlyNodes(3); if (remoteTranslog) { @@ -180,7 +366,7 @@ private void testPeerRecovery(boolean remoteTranslog, int numberOfIterations, bo ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); - Map indexStats = indexData(numberOfIterations, invokeFlush); + Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); client().admin() .indices() @@ -263,7 +449,7 @@ private void verifyRemoteStoreCleanup(boolean remoteTranslog) throws Exception { createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); } - indexData(5, randomBoolean()); + indexData(5, randomBoolean(), INDEX_NAME); String indexUUID = client().admin() .indices() .prepareGetSettings(INDEX_NAME) @@ -292,7 +478,7 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception { internalCluster().startDataOnlyNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l)); int numberOfIterations = randomIntBetween(5, 15); - indexData(numberOfIterations, true); + indexData(numberOfIterations, true, INDEX_NAME); String indexUUID = client().admin() .indices() .prepareGetSettings(INDEX_NAME) @@ -316,7 +502,7 @@ public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { internalCluster().startDataOnlyNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l)); int numberOfIterations = randomIntBetween(5, 15); - indexData(numberOfIterations, false); + indexData(numberOfIterations, false, INDEX_NAME); String indexUUID = client().admin() .indices() .prepareGetSettings(INDEX_NAME) From 8d7eccf080c318512f1731286be369168b44b2e7 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Thu, 6 Jul 2023 15:35:47 +0530 Subject: [PATCH 02/17] Add remote store restore ITs for 1-replica config, wildcards, exclusions, and default index list Signed-off-by: Bhumika Saini --- .../opensearch/remotestore/RemoteStoreIT.java | 361 +++++++++++++----- 1 file changed, 256 insertions(+), 105 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index b69bdf9660ffa..4483e68f9ffd6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -20,7 +20,6 @@ import org.opensearch.cluster.routing.RecoverySource; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.TimeValue; import org.opensearch.index.shard.RemoteStoreRefreshListener; import org.opensearch.indices.recovery.RecoveryState; import org.opensearch.plugins.Plugin; @@ -48,7 +47,8 @@ public class RemoteStoreIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; private static final String INDEX_NAME_WILDCARD = "remote-store-test-*"; - private static final String INDEX_NAMES = "remote-store-test-idx-1,remote-store-test-idx-2,remote-store-test-index-1,remote-store-test-index-2"; + private static final String INDEX_NAMES = + "remote-store-test-idx-1,remote-store-test-idx-2,remote-store-test-index-1,remote-store-test-index-2"; private static final String INDEX_NAMES_WILDCARD = "remote-store-test-idx-*,remote-store-test-index-*"; private static final String TOTAL_OPERATIONS = "total-operations"; private static final String REFRESHED_OR_FLUSHED_OPERATIONS = "refreshed-or-flushed-operations"; @@ -117,7 +117,13 @@ private void verifyRestoredData(Map indexStats, boolean checkTotal assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity) + 1); } - private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, boolean remoteTranslogEnabled, String indices, int replicaCount) { + private void prepareCluster( + int numClusterManagerNodes, + int numDataOnlyNodes, + boolean remoteTranslogEnabled, + String indices, + int replicaCount + ) { internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); internalCluster().startDataOnlyNodes(numDataOnlyNodes); for (String index : indices.split(",")) { @@ -160,7 +166,8 @@ private void testRestoreFlow(boolean remoteTranslog, int numberOfIterations, boo * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. * @throws IOException IO Exception. */ - private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws IOException, InterruptedException { + private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) + throws IOException { prepareCluster(1, 2, remoteTranslog, INDEX_NAME, 1); Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); @@ -168,49 +175,56 @@ private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(INDEX_NAME))); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); ensureRed(INDEX_NAME); - internalCluster().startDataOnlyNodes(2); - Thread.sleep(10000); assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); - ensureYellow(INDEX_NAME); + ensureGreen(INDEX_NAME); + assertEquals(0, getNumShards(INDEX_NAME).numReplicas); verifyRestoredData(indexStats, remoteTranslog, INDEX_NAME); } -// /** -// * Helper function to test restoring multiple indices from remote store when all the nodes housing the primary/replica drop. -// * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store. -// * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. -// * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. -// * @throws IOException IO Exception. -// */ -// private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws IOException { -// internalCluster().startDataOnlyNodes(3); -// if (remoteTranslog) { -// createIndex(INDEX_NAME, remoteTranslogIndexSettings(1)); -// } else { -// createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); -// } -// ensureYellowAndNoInitializingShards(INDEX_NAME); -// ensureGreen(INDEX_NAME); -// -// Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); -// -// internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); -// internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(INDEX_NAME))); -// assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); -// -// client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); -// ensureGreen(INDEX_NAME); -// -// if (remoteTranslog) { -// verifyRestoredData(indexStats, true, INDEX_NAME); -// } else { -// verifyRestoredData(indexStats, false, INDEX_NAME); -// } -// } + /** + * Helper function to test restoring multiple indices from remote store when all the nodes housing the primary/replica drop. + * @param remoteTranslog If true, Remote Translog Store is also enabled in addition to Remote Segment Store. + * @param numberOfIterations Number of times a refresh/flush should be invoked, followed by indexing some data. + * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. + * @throws IOException IO Exception. + */ + private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws IOException { + prepareCluster(1, 3, remoteTranslog, INDEX_NAMES, 1); + String[] indices = INDEX_NAMES.split(","); + Map> indicesStats = new HashMap<>(); + for (String index : indices) { + Map indexStats = indexData(numberOfIterations, invokeFlush, index); + indicesStats.put(index, indexStats); + } + + for (String index : indices) { + if (ClusterHealthStatus.RED.equals(ensureRed(index))) { + continue; + } + + if (ClusterHealthStatus.GREEN.equals(ensureRed(index))) { + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(index))); + } + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(index))); + } + + ensureRed(indices); + internalCluster().startDataOnlyNodes(3); + + assertAcked(client().admin().indices().prepareClose(indices)); + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAMES_WILDCARD.split(",")), PlainActionFuture.newFuture()); + ensureGreen(indices); + for (String index : indices) { + verifyRestoredData(indicesStats.get(index), remoteTranslog, index); + } + } /** * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. @@ -246,7 +260,7 @@ public void testRemoteSegmentStoreRestoreWithCommittedData() throws IOException testRestoreFlow(false, randomIntBetween(2, 5), true); } -// @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") /** * Simulates all data restored using Remote Translog Store. * @throws IOException IO Exception. @@ -281,80 +295,217 @@ public void testRemoteTranslogRestoreWithCommittedData() throws IOException { testRestoreFlow(true, randomIntBetween(2, 5), true); } -// /** -// * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. -// * @throws IOException IO Exception. -// */ -// public void testRSSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOException { -// testRestoreFlowBothPrimaryReplicasDown(false, 1, true); -// } -// -// /** -// * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. -// * @throws IOException IO Exception. -// */ -// public void testRSSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { -// testRestoreFlowBothPrimaryReplicasDown(false, 1, false); -// } + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") + /** + * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. + * @throws IOException IO Exception. + */ + public void testRSSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOException { + testRestoreFlowBothPrimaryReplicasDown(false, 1, true); + } + + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") + /** + * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. + * @throws IOException IO Exception. + */ + public void testRSSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { + testRestoreFlowBothPrimaryReplicasDown(false, 1, false); + } + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") /** * Simulates data restored until the refreshed data in Remote Segment Store * and data loss for the unrefreshed data. * @throws IOException IO Exception. */ - public void testRSSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOException, InterruptedException { + public void testRSSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOException { testRestoreFlowBothPrimaryReplicasDown(false, randomIntBetween(2, 5), false); } -// /** -// * Simulates data restored until the refreshed data in Remote Segment Store -// * and data loss for the unrefreshed data. -// * @throws IOException IO Exception. -// */ -// public void testRSSRestoreWithCommittedDataPrimaryReplicaDown() throws IOException { -// testRestoreFlowBothPrimaryReplicasDown(false, randomIntBetween(2, 5), true); -// } -// -//// @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") -// /** -// * Simulates all data restored using Remote Translog Store. -// * @throws IOException IO Exception. -// */ -// public void testRTSWithNoDataPostCommitPrimaryReplicaDown() throws IOException { -// testRestoreFlowBothPrimaryReplicasDown(true, 1, true); -// } -// -// /** -// * Simulates all data restored using Remote Translog Store. -// * @throws IOException IO Exception. -// */ -// public void testRTSWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { -// testRestoreFlowBothPrimaryReplicasDown(true, 1, false); -// } -// -// /** -// * Simulates refreshed data restored using Remote Segment Store -// * and unrefreshed data restored using Remote Translog Store. -// * @throws IOException IO Exception. -// */ -// public void testRTSWithRefreshedDataPrimaryReplicaDown() throws IOException { -// testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), false); -// } -// -// /** -// * Simulates refreshed data restored using Remote Segment Store -// * and unrefreshed data restored using Remote Translog Store. -// * @throws IOException IO Exception. -// */ -// public void testRTSWithCommittedDataPrimaryReplicaDown() throws IOException { -// testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), true); -// } - - - -// public void testRemoteTranslogRestoreWithCommittedDataIndexPattern() throws IOException { -// testRestoreFlow(true, randomIntBetween(2, 5), true); -// } + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") + /** + * Simulates data restored until the refreshed data in Remote Segment Store + * and data loss for the unrefreshed data. + * @throws IOException IO Exception. + */ + public void testRSSRestoreWithCommittedDataPrimaryReplicaDown() throws IOException { + testRestoreFlowBothPrimaryReplicasDown(false, randomIntBetween(2, 5), true); + } + + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") + /** + * Simulates all data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ + public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOException { + testRestoreFlowBothPrimaryReplicasDown(true, 1, true); + } + + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") + /** + * Simulates all data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ + public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { + testRestoreFlowBothPrimaryReplicasDown(true, 1, false); + } + + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ + public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOException { + testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), false); + } + + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store. + * @throws IOException IO Exception. + */ + public void testRTSRestoreWithCommittedDataPrimaryReplicaDown() throws IOException { + testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), true); + } + + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store + * for multiple indices matching a wildcard name pattern. + * @throws IOException IO Exception. + */ + public void testRTSRestoreWithCommittedDataMultipleIndicesPatterns() throws IOException { + testRestoreFlowMultipleIndices(true, randomIntBetween(2, 5), true); + } + + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store, + * with all remote-enabled red indices considered for the restore by default. + * @throws IOException IO Exception. + */ + public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOException { + prepareCluster(1, 3, true, INDEX_NAMES, 1); + String[] indices = INDEX_NAMES.split(","); + Map> indicesStats = new HashMap<>(); + for (String index : indices) { + Map indexStats = indexData(randomIntBetween(2, 5), true, index); + indicesStats.put(index, indexStats); + } + + for (String index : indices) { + if (ClusterHealthStatus.RED.equals(ensureRed(index))) { + continue; + } + + if (ClusterHealthStatus.GREEN.equals(ensureRed(index))) { + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(index))); + } + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(index))); + } + + ensureRed(indices); + internalCluster().startDataOnlyNodes(3); + + assertAcked(client().admin().indices().prepareClose(indices)); + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(new String[] {}), PlainActionFuture.newFuture()); + ensureGreen(indices); + + for (String index : indices) { + verifyRestoredData(indicesStats.get(index), true, index); + } + } + + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store, + * with only some of the remote-enabled red indices requested for the restore. + * @throws IOException IO Exception. + */ + public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOException { + prepareCluster(1, 3, true, INDEX_NAMES, 1); + String[] indices = INDEX_NAMES.split(","); + Map> indicesStats = new HashMap<>(); + for (String index : indices) { + Map indexStats = indexData(randomIntBetween(2, 5), true, index); + indicesStats.put(index, indexStats); + } + + for (String index : indices) { + if (ClusterHealthStatus.RED.equals(ensureRed(index))) { + continue; + } + + if (ClusterHealthStatus.GREEN.equals(ensureRed(index))) { + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(index))); + } + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(index))); + } + + ensureRed(indices); + internalCluster().startDataOnlyNodes(3); + + assertAcked(client().admin().indices().prepareClose(indices[0], indices[1])); + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indices[0], indices[1]), PlainActionFuture.newFuture()); + ensureGreen(indices[0], indices[1]); + verifyRestoredData(indicesStats.get(indices[0]), true, indices[0]); + verifyRestoredData(indicesStats.get(indices[1]), true, indices[1]); + ensureRed(indices[2], indices[3]); + } + + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store, + * with all remote-enabled red indices being considered for the restore + * except those matching the specified exclusion pattern. + * @throws IOException IO Exception. + */ + public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOException { + prepareCluster(1, 3, true, INDEX_NAMES, 1); + String[] indices = INDEX_NAMES.split(","); + Map> indicesStats = new HashMap<>(); + for (String index : indices) { + Map indexStats = indexData(randomIntBetween(2, 5), true, index); + indicesStats.put(index, indexStats); + } + + for (String index : indices) { + if (ClusterHealthStatus.RED.equals(ensureRed(index))) { + continue; + } + + if (ClusterHealthStatus.GREEN.equals(ensureRed(index))) { + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(index))); + } + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(index))); + } + + ensureRed(indices); + internalCluster().startDataOnlyNodes(3); + + assertAcked(client().admin().indices().prepareClose(indices[0], indices[1])); + client().admin() + .cluster() + .restoreRemoteStore(new RestoreRemoteStoreRequest().indices("*", "-remote-store-test-index-*"), PlainActionFuture.newFuture()); + ensureGreen(indices[0], indices[1]); + verifyRestoredData(indicesStats.get(indices[0]), true, indices[0]); + verifyRestoredData(indicesStats.get(indices[1]), true, indices[1]); + ensureRed(indices[2], indices[3]); + } private void testPeerRecovery(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws Exception { internalCluster().startDataOnlyNodes(3); From 5571101ed437adb437d24262e06f71f26de5476c Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Thu, 6 Jul 2023 23:32:01 +0530 Subject: [PATCH 03/17] Mute failing tests Signed-off-by: Bhumika Saini --- .../opensearch/remotestore/RemoteStoreIT.java | 38 +++++++++---------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 4483e68f9ffd6..e0175b6c98cc3 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -295,107 +295,107 @@ public void testRemoteTranslogRestoreWithCommittedData() throws IOException { testRestoreFlow(true, randomIntBetween(2, 5), true); } - // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") /** * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. * @throws IOException IO Exception. */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRSSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOException { testRestoreFlowBothPrimaryReplicasDown(false, 1, true); } - // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") /** * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. * @throws IOException IO Exception. */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRSSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { testRestoreFlowBothPrimaryReplicasDown(false, 1, false); } - // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") /** * Simulates data restored until the refreshed data in Remote Segment Store * and data loss for the unrefreshed data. * @throws IOException IO Exception. */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRSSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOException { testRestoreFlowBothPrimaryReplicasDown(false, randomIntBetween(2, 5), false); } - // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") /** * Simulates data restored until the refreshed data in Remote Segment Store * and data loss for the unrefreshed data. * @throws IOException IO Exception. */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRSSRestoreWithCommittedDataPrimaryReplicaDown() throws IOException { testRestoreFlowBothPrimaryReplicasDown(false, randomIntBetween(2, 5), true); } - // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") - // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") /** * Simulates all data restored using Remote Translog Store. * @throws IOException IO Exception. */ + // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOException { testRestoreFlowBothPrimaryReplicasDown(true, 1, true); } - // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") /** * Simulates all data restored using Remote Translog Store. * @throws IOException IO Exception. */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { testRestoreFlowBothPrimaryReplicasDown(true, 1, false); } - // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") /** * Simulates refreshed data restored using Remote Segment Store * and unrefreshed data restored using Remote Translog Store. * @throws IOException IO Exception. */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOException { testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), false); } - // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") /** * Simulates refreshed data restored using Remote Segment Store * and unrefreshed data restored using Remote Translog Store. * @throws IOException IO Exception. */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithCommittedDataPrimaryReplicaDown() throws IOException { testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), true); } - // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") /** * Simulates refreshed data restored using Remote Segment Store * and unrefreshed data restored using Remote Translog Store * for multiple indices matching a wildcard name pattern. * @throws IOException IO Exception. */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") public void testRTSRestoreWithCommittedDataMultipleIndicesPatterns() throws IOException { - testRestoreFlowMultipleIndices(true, randomIntBetween(2, 5), true); + testRestoreFlowMultipleIndices(true, 2, true); } - // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") /** * Simulates refreshed data restored using Remote Segment Store * and unrefreshed data restored using Remote Translog Store, * with all remote-enabled red indices considered for the restore by default. * @throws IOException IO Exception. */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOException { prepareCluster(1, 3, true, INDEX_NAMES, 1); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { - Map indexStats = indexData(randomIntBetween(2, 5), true, index); + Map indexStats = indexData(2, true, index); indicesStats.put(index, indexStats); } @@ -432,11 +432,11 @@ public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOExceptio * @throws IOException IO Exception. */ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOException { - prepareCluster(1, 3, true, INDEX_NAMES, 1); + prepareCluster(1, 3, true, INDEX_NAMES, 0); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { - Map indexStats = indexData(randomIntBetween(2, 5), true, index); + Map indexStats = indexData(2, true, index); indicesStats.put(index, indexStats); } @@ -445,10 +445,6 @@ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOExc continue; } - if (ClusterHealthStatus.GREEN.equals(ensureRed(index))) { - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(index))); - } - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(index))); } @@ -465,7 +461,6 @@ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOExc ensureRed(indices[2], indices[3]); } - // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") /** * Simulates refreshed data restored using Remote Segment Store * and unrefreshed data restored using Remote Translog Store, @@ -473,12 +468,13 @@ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOExc * except those matching the specified exclusion pattern. * @throws IOException IO Exception. */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOException { prepareCluster(1, 3, true, INDEX_NAMES, 1); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { - Map indexStats = indexData(randomIntBetween(2, 5), true, index); + Map indexStats = indexData(2, true, index); indicesStats.put(index, indexStats); } From c52d1577e2013e22ab4a8d7dd9124e00b9220c84 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Tue, 11 Jul 2023 16:27:07 +0530 Subject: [PATCH 04/17] Add restore IT for multiple shards Signed-off-by: Bhumika Saini --- .../opensearch/remotestore/RemoteStoreIT.java | 111 ++++-------------- 1 file changed, 24 insertions(+), 87 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index e0175b6c98cc3..0a66bbaf2c453 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -122,15 +122,16 @@ private void prepareCluster( int numDataOnlyNodes, boolean remoteTranslogEnabled, String indices, - int replicaCount + int replicaCount, + int shardCount ) { internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); internalCluster().startDataOnlyNodes(numDataOnlyNodes); for (String index : indices.split(",")) { if (remoteTranslogEnabled) { - createIndex(index, remoteTranslogIndexSettings(replicaCount)); + createIndex(index, remoteTranslogIndexSettings(replicaCount, shardCount)); } else { - createIndex(index, remoteStoreIndexSettings(replicaCount)); + createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); } ensureYellowAndNoInitializingShards(index); @@ -145,17 +146,19 @@ private void prepareCluster( * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. * @throws IOException IO Exception. */ - private void testRestoreFlow(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws IOException { - prepareCluster(0, 3, remoteTranslog, INDEX_NAME, 0); - + private void testRestoreFlow(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush, int shardCount) throws IOException { + prepareCluster(0, 3, remoteTranslog, INDEX_NAME, 0, shardCount); Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); + ensureRed(INDEX_NAME); assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); ensureGreen(INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); verifyRestoredData(indexStats, remoteTranslog, INDEX_NAME); } @@ -168,7 +171,7 @@ private void testRestoreFlow(boolean remoteTranslog, int numberOfIterations, boo */ private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws IOException { - prepareCluster(1, 2, remoteTranslog, INDEX_NAME, 1); + prepareCluster(1, 2, remoteTranslog, INDEX_NAME, 1, 1); Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); @@ -193,7 +196,7 @@ private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int * @throws IOException IO Exception. */ private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws IOException { - prepareCluster(1, 3, remoteTranslog, INDEX_NAMES, 1); + prepareCluster(1, 3, remoteTranslog, INDEX_NAMES, 1, 1); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { @@ -226,47 +229,13 @@ private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOf } } - /** - * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. - * @throws IOException IO Exception. - */ - public void testRemoteSegmentStoreRestoreWithNoDataPostCommit() throws IOException { - testRestoreFlow(false, 1, true); - } - - /** - * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. - * @throws IOException IO Exception. - */ - public void testRemoteSegmentStoreRestoreWithNoDataPostRefresh() throws IOException { - testRestoreFlow(false, 1, false); - } - - /** - * Simulates data restored until the refreshed data in Remote Segment Store - * and data loss for the unrefreshed data. - * @throws IOException IO Exception. - */ - public void testRemoteSegmentStoreRestoreWithRefreshedData() throws IOException { - testRestoreFlow(false, randomIntBetween(2, 5), false); - } - - /** - * Simulates data restored until the refreshed data in Remote Segment Store - * and data loss for the unrefreshed data. - * @throws IOException IO Exception. - */ - public void testRemoteSegmentStoreRestoreWithCommittedData() throws IOException { - testRestoreFlow(false, randomIntBetween(2, 5), true); - } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") /** * Simulates all data restored using Remote Translog Store. * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithNoDataPostCommit() throws IOException { - testRestoreFlow(true, 1, true); + testRestoreFlow(true, 1, true, 1); } /** @@ -274,7 +243,7 @@ public void testRemoteTranslogRestoreWithNoDataPostCommit() throws IOException { * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithNoDataPostRefresh() throws IOException { - testRestoreFlow(true, 1, false); + testRestoreFlow(true, 1, false, 1); } /** @@ -283,7 +252,7 @@ public void testRemoteTranslogRestoreWithNoDataPostRefresh() throws IOException * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithRefreshedData() throws IOException { - testRestoreFlow(true, randomIntBetween(2, 5), false); + testRestoreFlow(true, randomIntBetween(2, 5), false, 1); } /** @@ -292,45 +261,7 @@ public void testRemoteTranslogRestoreWithRefreshedData() throws IOException { * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithCommittedData() throws IOException { - testRestoreFlow(true, randomIntBetween(2, 5), true); - } - - /** - * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. - * @throws IOException IO Exception. - */ - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") - public void testRSSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(false, 1, true); - } - - /** - * Simulates full data loss due to unrefreshed data, with no data restored from Remote Segment Store. - * @throws IOException IO Exception. - */ - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") - public void testRSSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(false, 1, false); - } - - /** - * Simulates data restored until the refreshed data in Remote Segment Store - * and data loss for the unrefreshed data. - * @throws IOException IO Exception. - */ - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") - public void testRSSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(false, randomIntBetween(2, 5), false); - } - - /** - * Simulates data restored until the refreshed data in Remote Segment Store - * and data loss for the unrefreshed data. - * @throws IOException IO Exception. - */ - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") - public void testRSSRestoreWithCommittedDataPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(false, randomIntBetween(2, 5), true); + testRestoreFlow(true, randomIntBetween(2, 5), true, 1); } /** @@ -391,7 +322,7 @@ public void testRTSRestoreWithCommittedDataMultipleIndicesPatterns() throws IOEx */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOException { - prepareCluster(1, 3, true, INDEX_NAMES, 1); + prepareCluster(1, 3, true, INDEX_NAMES, 1, 1); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { @@ -432,7 +363,7 @@ public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOExceptio * @throws IOException IO Exception. */ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOException { - prepareCluster(1, 3, true, INDEX_NAMES, 0); + prepareCluster(1, 3, true, INDEX_NAMES, 0, 1); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { @@ -470,7 +401,7 @@ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOExc */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOException { - prepareCluster(1, 3, true, INDEX_NAMES, 1); + prepareCluster(1, 3, true, INDEX_NAMES, 1, 1); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { @@ -503,6 +434,12 @@ public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOExc ensureRed(indices[2], indices[3]); } + public void testRTSRestoreWithCommittedDataMultipleShards() throws IOException { + testRestoreFlow(true, 2, true, 2); + } + + // TODO: Restore flow - index aliases + private void testPeerRecovery(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws Exception { internalCluster().startDataOnlyNodes(3); if (remoteTranslog) { From b7ec9082139274ae6841f1f55b096c1933aad212 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Tue, 11 Jul 2023 18:34:31 +0530 Subject: [PATCH 05/17] Fix new tests for multiple shards and selective red indices Signed-off-by: Bhumika Saini --- .../opensearch/remotestore/RemoteStoreIT.java | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 0a66bbaf2c453..5756e222dd26c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -47,8 +47,7 @@ public class RemoteStoreIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; private static final String INDEX_NAME_WILDCARD = "remote-store-test-*"; - private static final String INDEX_NAMES = - "remote-store-test-idx-1,remote-store-test-idx-2,remote-store-test-index-1,remote-store-test-index-2"; + private static final String INDEX_NAMES = "remote-store-test-1,remote-store-test-2,remote-store-test-index-1,remote-store-test-index-2"; private static final String INDEX_NAMES_WILDCARD = "remote-store-test-idx-*,remote-store-test-index-*"; private static final String TOTAL_OPERATIONS = "total-operations"; private static final String REFRESHED_OR_FLUSHED_OPERATIONS = "refreshed-or-flushed-operations"; @@ -77,11 +76,20 @@ private IndexResponse indexSingleDoc() { .get(); } + private IndexResponse indexSingleDoc(String indexName) { + return client().prepareIndex(indexName) + .setId(UUIDs.randomBase64UUID()) + .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) + .get(); + } + private Map indexData(int numberOfIterations, boolean invokeFlush, String index) { long totalOperations = 0; long refreshedOrFlushedOperations = 0; long maxSeqNo = -1; long maxSeqNoRefreshedOrFlushed = -1; + int shardId = 0; + Map indexingStats = new HashMap<>(); for (int i = 0; i < numberOfIterations; i++) { if (invokeFlush) { flush(index); @@ -89,15 +97,18 @@ private Map indexData(int numberOfIterations, boolean invokeFlush, refresh(index); } maxSeqNoRefreshedOrFlushed = maxSeqNo; + indexingStats.put(MAX_SEQ_NO_REFRESHED_OR_FLUSHED + "-shard-" + shardId, maxSeqNoRefreshedOrFlushed); refreshedOrFlushedOperations = totalOperations; int numberOfOperations = randomIntBetween(20, 50); for (int j = 0; j < numberOfOperations; j++) { - IndexResponse response = indexSingleDoc(); + IndexResponse response = INDEX_NAME.equals(index) ? indexSingleDoc() : indexSingleDoc(index); maxSeqNo = response.getSeqNo(); + shardId = response.getShardId().id(); + indexingStats.put(MAX_SEQ_NO_TOTAL + "-shard-" + shardId, maxSeqNo); } totalOperations += numberOfOperations; } - Map indexingStats = new HashMap<>(); + indexingStats.put(TOTAL_OPERATIONS, totalOperations); indexingStats.put(REFRESHED_OR_FLUSHED_OPERATIONS, refreshedOrFlushedOperations); indexingStats.put(MAX_SEQ_NO_TOTAL, maxSeqNo); @@ -112,7 +123,7 @@ private void verifyRestoredData(Map indexStats, boolean checkTotal ensureGreen(indexName); assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity)); IndexResponse response = indexSingleDoc(); - assertEquals(indexStats.get(maxSeqNoGranularity) + 1, response.getSeqNo()); + assertEquals(indexStats.get(maxSeqNoGranularity + "-shard-" + response.getShardId().id()) + 1, response.getSeqNo()); refresh(indexName); assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity) + 1); } @@ -205,11 +216,12 @@ private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOf } for (String index : indices) { - if (ClusterHealthStatus.RED.equals(ensureRed(index))) { + ClusterHealthStatus indexHealth = ensureRed(index); + if (ClusterHealthStatus.RED.equals(indexHealth)) { continue; } - if (ClusterHealthStatus.GREEN.equals(ensureRed(index))) { + if (ClusterHealthStatus.GREEN.equals(indexHealth)) { internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(index))); } @@ -229,11 +241,11 @@ private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOf } } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") /** * Simulates all data restored using Remote Translog Store. * @throws IOException IO Exception. */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") public void testRemoteTranslogRestoreWithNoDataPostCommit() throws IOException { testRestoreFlow(true, 1, true, 1); } @@ -434,10 +446,26 @@ public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOExc ensureRed(indices[2], indices[3]); } + /** + * Simulates refreshed data restored using Remote Segment Store + * and unrefreshed data restored using Remote Translog Store, + * when the index has multiple shards. + * @throws IOException IO Exception. + */ public void testRTSRestoreWithCommittedDataMultipleShards() throws IOException { testRestoreFlow(true, 2, true, 2); } + /** + * Simulates no-op restore from remote store, + * when the index has no data. + * @throws IOException IO Exception. + */ + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") + public void testRTSRestoreNoData() throws IOException { + testRestoreFlow(true, 0, true, 2); + } + // TODO: Restore flow - index aliases private void testPeerRecovery(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws Exception { From 48323deded9f5cd5bda629707b3d7088f02ad9b3 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Tue, 11 Jul 2023 18:57:36 +0530 Subject: [PATCH 06/17] Fix wildcard, verifyRestoredData Signed-off-by: Bhumika Saini --- .../java/org/opensearch/remotestore/RemoteStoreIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 5756e222dd26c..0fa8c71f62baa 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -48,7 +48,7 @@ public class RemoteStoreIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; private static final String INDEX_NAME_WILDCARD = "remote-store-test-*"; private static final String INDEX_NAMES = "remote-store-test-1,remote-store-test-2,remote-store-test-index-1,remote-store-test-index-2"; - private static final String INDEX_NAMES_WILDCARD = "remote-store-test-idx-*,remote-store-test-index-*"; + private static final String INDEX_NAMES_WILDCARD = "remote-store-test-*,remote-store-test-index-*"; private static final String TOTAL_OPERATIONS = "total-operations"; private static final String REFRESHED_OR_FLUSHED_OPERATIONS = "refreshed-or-flushed-operations"; private static final String MAX_SEQ_NO_TOTAL = "max-seq-no-total"; @@ -122,7 +122,7 @@ private void verifyRestoredData(Map indexStats, boolean checkTotal ensureYellowAndNoInitializingShards(indexName); ensureGreen(indexName); assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity)); - IndexResponse response = indexSingleDoc(); + IndexResponse response = INDEX_NAME.equals(indexName) ? indexSingleDoc() : indexSingleDoc(indexName); assertEquals(indexStats.get(maxSeqNoGranularity + "-shard-" + response.getShardId().id()) + 1, response.getSeqNo()); refresh(indexName); assertHitCount(client().prepareSearch(indexName).setSize(0).get(), indexStats.get(statsGranularity) + 1); From 88e6cfbd5e8235d9936ad4fcb74d28b0ceac23e6 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Tue, 11 Jul 2023 19:04:41 +0530 Subject: [PATCH 07/17] Change index names to match different wildcards Signed-off-by: Bhumika Saini --- .../java/org/opensearch/remotestore/RemoteStoreIT.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 0fa8c71f62baa..8a0374d7caebd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -46,9 +46,8 @@ public class RemoteStoreIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; - private static final String INDEX_NAME_WILDCARD = "remote-store-test-*"; - private static final String INDEX_NAMES = "remote-store-test-1,remote-store-test-2,remote-store-test-index-1,remote-store-test-index-2"; - private static final String INDEX_NAMES_WILDCARD = "remote-store-test-*,remote-store-test-index-*"; + private static final String INDEX_NAMES = "test-remote-store-1,test-remote-store-2,remote-store-test-index-1,remote-store-test-index-2"; + private static final String INDEX_NAMES_WILDCARD = "test-remote-store-*,remote-store-test-index-*"; private static final String TOTAL_OPERATIONS = "total-operations"; private static final String REFRESHED_OR_FLUSHED_OPERATIONS = "refreshed-or-flushed-operations"; private static final String MAX_SEQ_NO_TOTAL = "max-seq-no-total"; From 525b4d21d10c448c88207e1c495eae839916bee4 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Wed, 12 Jul 2023 09:14:47 +0530 Subject: [PATCH 08/17] Empty commit Signed-off-by: Bhumika Saini From cdcf61515bc04960c2c621cb50d223acb23c6f0d Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Wed, 12 Jul 2023 10:22:01 +0530 Subject: [PATCH 09/17] Remove testRTSRestoreWithCommittedDataMultipleShards, Randomize shard count in restore ITs Signed-off-by: Bhumika Saini --- .../opensearch/remotestore/RemoteStoreIT.java | 61 ++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 8a0374d7caebd..ac0368f0183ac 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -179,11 +179,11 @@ private void testRestoreFlow(boolean remoteTranslog, int numberOfIterations, boo * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. * @throws IOException IO Exception. */ - private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) + private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush, int shardCount) throws IOException { - prepareCluster(1, 2, remoteTranslog, INDEX_NAME, 1, 1); - + prepareCluster(1, 2, remoteTranslog, INDEX_NAME, 1, shardCount); Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(INDEX_NAME))); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); @@ -194,6 +194,7 @@ private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int client().admin().cluster().restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAME), PlainActionFuture.newFuture()); ensureGreen(INDEX_NAME); + assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); assertEquals(0, getNumShards(INDEX_NAME).numReplicas); verifyRestoredData(indexStats, remoteTranslog, INDEX_NAME); } @@ -205,13 +206,15 @@ private void testRestoreFlowBothPrimaryReplicasDown(boolean remoteTranslog, int * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. * @throws IOException IO Exception. */ - private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush) throws IOException { - prepareCluster(1, 3, remoteTranslog, INDEX_NAMES, 1, 1); + private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOfIterations, boolean invokeFlush, int shardCount) + throws IOException { + prepareCluster(1, 3, remoteTranslog, INDEX_NAMES, 1, shardCount); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { Map indexStats = indexData(numberOfIterations, invokeFlush, index); indicesStats.put(index, indexStats); + assertEquals(shardCount, getNumShards(index).totalNumShards); } for (String index : indices) { @@ -236,6 +239,7 @@ private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOf .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(INDEX_NAMES_WILDCARD.split(",")), PlainActionFuture.newFuture()); ensureGreen(indices); for (String index : indices) { + assertEquals(shardCount, getNumShards(index).totalNumShards); verifyRestoredData(indicesStats.get(index), remoteTranslog, index); } } @@ -246,7 +250,7 @@ private void testRestoreFlowMultipleIndices(boolean remoteTranslog, int numberOf */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") public void testRemoteTranslogRestoreWithNoDataPostCommit() throws IOException { - testRestoreFlow(true, 1, true, 1); + testRestoreFlow(true, 1, true, randomIntBetween(1, 5)); } /** @@ -254,7 +258,7 @@ public void testRemoteTranslogRestoreWithNoDataPostCommit() throws IOException { * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithNoDataPostRefresh() throws IOException { - testRestoreFlow(true, 1, false, 1); + testRestoreFlow(true, 1, false, randomIntBetween(1, 5)); } /** @@ -263,7 +267,7 @@ public void testRemoteTranslogRestoreWithNoDataPostRefresh() throws IOException * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithRefreshedData() throws IOException { - testRestoreFlow(true, randomIntBetween(2, 5), false, 1); + testRestoreFlow(true, randomIntBetween(2, 5), false, randomIntBetween(1, 5)); } /** @@ -272,7 +276,7 @@ public void testRemoteTranslogRestoreWithRefreshedData() throws IOException { * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithCommittedData() throws IOException { - testRestoreFlow(true, randomIntBetween(2, 5), true, 1); + testRestoreFlow(true, randomIntBetween(2, 5), true, randomIntBetween(1, 5)); } /** @@ -282,7 +286,7 @@ public void testRemoteTranslogRestoreWithCommittedData() throws IOException { // @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(true, 1, true); + testRestoreFlowBothPrimaryReplicasDown(true, 1, true, randomIntBetween(1, 5)); } /** @@ -291,7 +295,7 @@ public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws IOExce */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(true, 1, false); + testRestoreFlowBothPrimaryReplicasDown(true, 1, false, randomIntBetween(1, 5)); } /** @@ -301,7 +305,7 @@ public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws IOExc */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), false); + testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), false, randomIntBetween(1, 5)); } /** @@ -311,7 +315,7 @@ public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws IOExcepti */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8479") public void testRTSRestoreWithCommittedDataPrimaryReplicaDown() throws IOException { - testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), true); + testRestoreFlowBothPrimaryReplicasDown(true, randomIntBetween(2, 5), true, randomIntBetween(1, 5)); } /** @@ -322,7 +326,7 @@ public void testRTSRestoreWithCommittedDataPrimaryReplicaDown() throws IOExcepti */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") public void testRTSRestoreWithCommittedDataMultipleIndicesPatterns() throws IOException { - testRestoreFlowMultipleIndices(true, 2, true); + testRestoreFlowMultipleIndices(true, 2, true, randomIntBetween(1, 5)); } /** @@ -333,12 +337,14 @@ public void testRTSRestoreWithCommittedDataMultipleIndicesPatterns() throws IOEx */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOException { - prepareCluster(1, 3, true, INDEX_NAMES, 1, 1); + int shardCount = randomIntBetween(1, 5); + prepareCluster(1, 3, true, INDEX_NAMES, 1, shardCount); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { Map indexStats = indexData(2, true, index); indicesStats.put(index, indexStats); + assertEquals(shardCount, getNumShards(index).totalNumShards); } for (String index : indices) { @@ -363,6 +369,7 @@ public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOExceptio ensureGreen(indices); for (String index : indices) { + assertEquals(shardCount, getNumShards(index).totalNumShards); verifyRestoredData(indicesStats.get(index), true, index); } } @@ -374,12 +381,14 @@ public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws IOExceptio * @throws IOException IO Exception. */ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOException { - prepareCluster(1, 3, true, INDEX_NAMES, 0, 1); + int shardCount = randomIntBetween(1, 5); + prepareCluster(1, 3, true, INDEX_NAMES, 0, shardCount); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { Map indexStats = indexData(2, true, index); indicesStats.put(index, indexStats); + assertEquals(shardCount, getNumShards(index).totalNumShards); } for (String index : indices) { @@ -398,7 +407,9 @@ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOExc .cluster() .restoreRemoteStore(new RestoreRemoteStoreRequest().indices(indices[0], indices[1]), PlainActionFuture.newFuture()); ensureGreen(indices[0], indices[1]); + assertEquals(shardCount, getNumShards(indices[0]).totalNumShards); verifyRestoredData(indicesStats.get(indices[0]), true, indices[0]); + assertEquals(shardCount, getNumShards(indices[1]).totalNumShards); verifyRestoredData(indicesStats.get(indices[1]), true, indices[1]); ensureRed(indices[2], indices[3]); } @@ -412,12 +423,14 @@ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws IOExc */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480") public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOException { - prepareCluster(1, 3, true, INDEX_NAMES, 1, 1); + int shardCount = randomIntBetween(1, 5); + prepareCluster(1, 3, true, INDEX_NAMES, 1, shardCount); String[] indices = INDEX_NAMES.split(","); Map> indicesStats = new HashMap<>(); for (String index : indices) { Map indexStats = indexData(2, true, index); indicesStats.put(index, indexStats); + assertEquals(shardCount, getNumShards(index).totalNumShards); } for (String index : indices) { @@ -440,21 +453,13 @@ public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws IOExc .cluster() .restoreRemoteStore(new RestoreRemoteStoreRequest().indices("*", "-remote-store-test-index-*"), PlainActionFuture.newFuture()); ensureGreen(indices[0], indices[1]); + assertEquals(shardCount, getNumShards(indices[0]).totalNumShards); verifyRestoredData(indicesStats.get(indices[0]), true, indices[0]); + assertEquals(shardCount, getNumShards(indices[1]).totalNumShards); verifyRestoredData(indicesStats.get(indices[1]), true, indices[1]); ensureRed(indices[2], indices[3]); } - /** - * Simulates refreshed data restored using Remote Segment Store - * and unrefreshed data restored using Remote Translog Store, - * when the index has multiple shards. - * @throws IOException IO Exception. - */ - public void testRTSRestoreWithCommittedDataMultipleShards() throws IOException { - testRestoreFlow(true, 2, true, 2); - } - /** * Simulates no-op restore from remote store, * when the index has no data. @@ -462,7 +467,7 @@ public void testRTSRestoreWithCommittedDataMultipleShards() throws IOException { */ @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6188") public void testRTSRestoreNoData() throws IOException { - testRestoreFlow(true, 0, true, 2); + testRestoreFlow(true, 0, true, randomIntBetween(1, 5)); } // TODO: Restore flow - index aliases From 48b9111ae7ee93463e590c009f8c4b9aec94a380 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Wed, 12 Jul 2023 12:45:05 +0530 Subject: [PATCH 10/17] Skip flaky tests testStaleCommitDeletionWithInvokeFlush and testStaleCommitDeletionWithoutInvokeFlush Signed-off-by: Bhumika Saini --- .../java/org/opensearch/remotestore/RemoteStoreIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index ac0368f0183ac..490fbcf8b42fc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -590,6 +590,7 @@ public void testRemoteTranslogCleanup() throws Exception { verifyRemoteStoreCleanup(true); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8658") public void testStaleCommitDeletionWithInvokeFlush() throws Exception { internalCluster().startDataOnlyNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l)); @@ -614,6 +615,7 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception { }, 30, TimeUnit.SECONDS); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8658") public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { internalCluster().startDataOnlyNodes(3); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l)); From 9f93f0e4175993c92a94f6de44e51c63cf80c2e6 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Wed, 12 Jul 2023 13:55:47 +0530 Subject: [PATCH 11/17] Empty commit Signed-off-by: Bhumika Saini From 6f3b206e0616a66fd749972e1ed895810a97c9a1 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Thu, 20 Jul 2023 01:40:54 +0530 Subject: [PATCH 12/17] Override expectEmptyRetentionLeases in RemoteStoreRecoverySource Signed-off-by: Bhumika Saini --- .../java/org/opensearch/cluster/routing/RecoverySource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java index aafc37fba9e85..53b7e27680d58 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java @@ -490,6 +490,10 @@ public int hashCode() { return Objects.hash(restoreUUID, index, version); } + @Override + public boolean expectEmptyRetentionLeases() { + return false; + } } /** From a0e0e13d54fa6204a3360000c39c64d5aa0ec439 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Thu, 20 Jul 2023 10:32:54 +0530 Subject: [PATCH 13/17] Empty commit Signed-off-by: Bhumika Saini From 74931f6a78472052122421d7f998b3a88d63b310 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Thu, 20 Jul 2023 15:12:36 +0530 Subject: [PATCH 14/17] Randomize RTS repo selection be same/different than RSS repo Signed-off-by: Bhumika Saini --- .../remotestore/RemoteStoreBaseIntegTestCase.java | 9 ++++++++- .../java/org/opensearch/remotestore/RemoteStoreIT.java | 7 ------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index eb8913591b38b..9bccb17e4a47e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -11,6 +11,7 @@ import org.junit.After; import org.opensearch.action.index.IndexResponse; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.Randomness; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -32,6 +33,7 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected static final String REPOSITORY_NAME = "test-remore-store-repo"; + protected static final String REPOSITORY_2_NAME = "test-remore-store-repo-2"; protected static final int SHARD_COUNT = 1; protected static final int REPLICA_COUNT = 1; @@ -95,10 +97,11 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas, long totalFiel } protected Settings remoteTranslogIndexSettings(int numberOfReplicas, int numberOfShards) { + boolean sameRepoForRSSAndRTS = Randomness.get().nextBoolean(); return Settings.builder() .put(remoteStoreIndexSettings(numberOfReplicas, numberOfShards)) .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, sameRepoForRSSAndRTS ? REPOSITORY_NAME : REPOSITORY_2_NAME) .build(); } @@ -110,6 +113,9 @@ protected void putRepository(Path path) { assertAcked( clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", path)) ); + assertAcked( + clusterAdmin().preparePutRepository(REPOSITORY_2_NAME).setType("fs").setSettings(Settings.builder().put("location", path)) + ); } protected void setupRepo() { @@ -121,6 +127,7 @@ protected void setupRepo() { @After public void teardown() { assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); + assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_2_NAME)); } public int getFileCount(Path path) throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 490fbcf8b42fc..4fcaeabae05fe 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -75,13 +75,6 @@ private IndexResponse indexSingleDoc() { .get(); } - private IndexResponse indexSingleDoc(String indexName) { - return client().prepareIndex(indexName) - .setId(UUIDs.randomBase64UUID()) - .setSource(randomAlphaOfLength(5), randomAlphaOfLength(5)) - .get(); - } - private Map indexData(int numberOfIterations, boolean invokeFlush, String index) { long totalOperations = 0; long refreshedOrFlushedOperations = 0; From 990b1602f81a71f32679c997046c3b9d9555ffb9 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Thu, 20 Jul 2023 16:59:45 +0530 Subject: [PATCH 15/17] Fix tests failing due to missing repo Signed-off-by: Bhumika Saini --- .../RemoteStoreBaseIntegTestCase.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index 9bccb17e4a47e..0b3ceac176193 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -11,7 +11,6 @@ import org.junit.After; import org.opensearch.action.index.IndexResponse; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.Randomness; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -36,8 +35,8 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected static final String REPOSITORY_2_NAME = "test-remore-store-repo-2"; protected static final int SHARD_COUNT = 1; protected static final int REPLICA_COUNT = 1; - protected Path absolutePath; + protected Path absolutePath2; @Override protected boolean addMockInternalEngine() { @@ -97,7 +96,7 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas, long totalFiel } protected Settings remoteTranslogIndexSettings(int numberOfReplicas, int numberOfShards) { - boolean sameRepoForRSSAndRTS = Randomness.get().nextBoolean(); + boolean sameRepoForRSSAndRTS = randomBoolean(); return Settings.builder() .put(remoteStoreIndexSettings(numberOfReplicas, numberOfShards)) .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, true) @@ -110,18 +109,19 @@ protected Settings remoteTranslogIndexSettings(int numberOfReplicas) { } protected void putRepository(Path path) { - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", path)) - ); - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_2_NAME).setType("fs").setSettings(Settings.builder().put("location", path)) - ); + putRepository(path, REPOSITORY_NAME); + } + + protected void putRepository(Path path, String repoName) { + assertAcked(clusterAdmin().preparePutRepository(repoName).setType("fs").setSettings(Settings.builder().put("location", path))); } protected void setupRepo() { internalCluster().startClusterManagerOnlyNode(); absolutePath = randomRepoPath().toAbsolutePath(); putRepository(absolutePath); + absolutePath2 = randomRepoPath().toAbsolutePath(); + putRepository(absolutePath2, REPOSITORY_2_NAME); } @After From 6fde387e51f35c86054f011c67b48c8b4db8c3bc Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Thu, 20 Jul 2023 17:14:57 +0530 Subject: [PATCH 16/17] Add TODO to update RemoteStoreRecoverySource#expectEmptyRetentionLeases once #8795 is resolved Signed-off-by: Bhumika Saini --- .../java/org/opensearch/cluster/routing/RecoverySource.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java index 53b7e27680d58..5cef46689ffc7 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java +++ b/server/src/main/java/org/opensearch/cluster/routing/RecoverySource.java @@ -490,6 +490,9 @@ public int hashCode() { return Objects.hash(restoreUUID, index, version); } + // TODO: This override should be removed/be updated to return "true", + // i.e. we expect no retention leases, once the following issue is fixed: + // https://github.com/opensearch-project/OpenSearch/issues/8795 @Override public boolean expectEmptyRetentionLeases() { return false; From dd1968d874757198a0cede2964214b4421a18d05 Mon Sep 17 00:00:00 2001 From: Bhumika Saini Date: Thu, 20 Jul 2023 18:22:53 +0530 Subject: [PATCH 17/17] Pass testPrimaryTermValidation() Signed-off-by: Bhumika Saini --- .../org/opensearch/remotestore/PrimaryTermValidationIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java index 6691da81f057d..d1fdde30a9537 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java @@ -69,6 +69,8 @@ public void testPrimaryTermValidation() throws Exception { assertAcked( clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) ); + absolutePath2 = randomRepoPath().toAbsolutePath(); + putRepository(absolutePath2, REPOSITORY_2_NAME); // Start data nodes and create index internalCluster().startDataOnlyNodes(2, clusterSettings);