From 40716ea7f6cbc44067e12d4c40314c7fa6222802 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Fri, 19 Jan 2024 16:57:33 +0530 Subject: [PATCH] Allow empty translog for ITs testing no ingestion post refresh/commit (#11946) Signed-off-by: bansvaru Signed-off-by: Shivansh Arora --- .../RemoteStoreBaseIntegTestCase.java | 9 +++++++ .../remotestore/RemoteStoreRestoreIT.java | 27 ++++++++++--------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index d23e634bb3368..751de66a97806 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -84,6 +84,10 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { ); protected Map indexData(int numberOfIterations, boolean invokeFlush, String index) { + return indexData(numberOfIterations, invokeFlush, false, index); + } + + protected Map indexData(int numberOfIterations, boolean invokeFlush, boolean emptyTranslog, String index) { long totalOperations = 0; long refreshedOrFlushedOperations = 0; long maxSeqNo = -1; @@ -96,6 +100,11 @@ protected Map indexData(int numberOfIterations, boolean invokeFlus } else { refresh(index); } + + // skip indexing if last iteration as we dont want to have any data in remote translog + if (emptyTranslog && i == numberOfIterations - 1) { + continue; + } maxSeqNoRefreshedOrFlushed = maxSeqNo; indexingStats.put(MAX_SEQ_NO_REFRESHED_OR_FLUSHED + "-shard-" + shardId, maxSeqNoRefreshedOrFlushed); refreshedOrFlushedOperations = totalOperations; diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java index 7626e3dba6424..72296a1c01a24 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java @@ -44,7 +44,7 @@ public class RemoteStoreRestoreIT extends BaseRemoteStoreRestoreIT { * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithNoDataPostCommit() throws Exception { - testRestoreFlow(1, true, randomIntBetween(1, 5)); + testRestoreFlow(1, true, true, randomIntBetween(1, 5)); } /** @@ -52,7 +52,7 @@ public void testRemoteTranslogRestoreWithNoDataPostCommit() throws Exception { * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithNoDataPostRefresh() throws Exception { - testRestoreFlow(1, false, randomIntBetween(1, 5)); + testRestoreFlow(1, false, true, randomIntBetween(1, 5)); } /** @@ -61,7 +61,7 @@ public void testRemoteTranslogRestoreWithNoDataPostRefresh() throws Exception { * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithRefreshedData() throws Exception { - testRestoreFlow(randomIntBetween(2, 5), false, randomIntBetween(1, 5)); + testRestoreFlow(randomIntBetween(2, 5), false, false, randomIntBetween(1, 5)); } /** @@ -70,7 +70,7 @@ public void testRemoteTranslogRestoreWithRefreshedData() throws Exception { * @throws IOException IO Exception. */ public void testRemoteTranslogRestoreWithCommittedData() throws Exception { - testRestoreFlow(randomIntBetween(2, 5), true, randomIntBetween(1, 5)); + testRestoreFlow(randomIntBetween(2, 5), true, false, randomIntBetween(1, 5)); } /** @@ -78,7 +78,7 @@ public void testRemoteTranslogRestoreWithCommittedData() throws Exception { * @throws IOException IO Exception. */ public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws Exception { - testRestoreFlowBothPrimaryReplicasDown(1, true, randomIntBetween(1, 5)); + testRestoreFlowBothPrimaryReplicasDown(1, true, true, randomIntBetween(1, 5)); } /** @@ -86,7 +86,7 @@ public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws Except * @throws IOException IO Exception. */ public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws Exception { - testRestoreFlowBothPrimaryReplicasDown(1, false, randomIntBetween(1, 5)); + testRestoreFlowBothPrimaryReplicasDown(1, false, true, randomIntBetween(1, 5)); } /** @@ -95,7 +95,7 @@ public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws Excep * @throws IOException IO Exception. */ public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws Exception { - testRestoreFlowBothPrimaryReplicasDown(randomIntBetween(2, 5), false, randomIntBetween(1, 5)); + testRestoreFlowBothPrimaryReplicasDown(randomIntBetween(2, 5), false, false, randomIntBetween(1, 5)); } /** @@ -104,7 +104,7 @@ public void testRTSRestoreWithRefreshedDataPrimaryReplicaDown() throws Exception * @throws IOException IO Exception. */ public void testRTSRestoreWithCommittedDataPrimaryReplicaDown() throws Exception { - testRestoreFlowBothPrimaryReplicasDown(randomIntBetween(2, 5), true, randomIntBetween(1, 5)); + testRestoreFlowBothPrimaryReplicasDown(randomIntBetween(2, 5), true, false, randomIntBetween(1, 5)); } private void restoreAndVerify(int shardCount, int replicaCount, Map indexStats) throws Exception { @@ -122,9 +122,9 @@ private void restoreAndVerify(int shardCount, int replicaCount, Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); + Map indexStats = indexData(numberOfIterations, invokeFlush, emptyTranslog, INDEX_NAME); assertEquals(shardCount, getNumShards(INDEX_NAME).totalNumShards); assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), indexStats.get(REFRESHED_OR_FLUSHED_OPERATIONS)); @@ -141,9 +141,10 @@ private void testRestoreFlow(int numberOfIterations, boolean invokeFlush, int sh * @param invokeFlush If true, a flush is invoked. Otherwise, a refresh is invoked. * @throws IOException IO Exception. */ - private void testRestoreFlowBothPrimaryReplicasDown(int numberOfIterations, boolean invokeFlush, int shardCount) throws Exception { + private void testRestoreFlowBothPrimaryReplicasDown(int numberOfIterations, boolean invokeFlush, boolean emptyTranslog, int shardCount) + throws Exception { prepareCluster(1, 2, INDEX_NAME, 1, shardCount); - Map indexStats = indexData(numberOfIterations, invokeFlush, INDEX_NAME); + Map indexStats = indexData(numberOfIterations, invokeFlush, emptyTranslog, INDEX_NAME); assertEquals(shardCount * 2, getNumShards(INDEX_NAME).totalNumShards); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName(INDEX_NAME))); @@ -391,7 +392,7 @@ public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws Excep * @throws IOException IO Exception. */ public void testRTSRestoreDataOnlyInTranslog() throws Exception { - testRestoreFlow(0, true, randomIntBetween(1, 5)); + testRestoreFlow(0, true, false, randomIntBetween(1, 5)); } public void testRateLimitedRemoteDownloads() throws Exception {