From b9bcad1176d07ca4e9b77aef1b03a73efe36e112 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Mon, 23 Oct 2023 23:27:36 +0530 Subject: [PATCH] Add integ tests Signed-off-by: Sachin Kale --- .../remotestore/RemoteStoreRestoreIT.java | 77 +++++++++++++++++++ .../opensearch/index/shard/StoreRecovery.java | 8 +- .../cluster/RestRestoreRemoteStoreAction.java | 1 + 3 files changed, 82 insertions(+), 4 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java index 7626e3dba6424..bfd8b749ed209 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java @@ -12,17 +12,21 @@ import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreResponse; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.util.FileSystemUtils; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Locale; @@ -30,6 +34,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -234,6 +239,78 @@ public void testRestoreFlowNoRedIndex() throws Exception { verifyRestoredData(indexStats, INDEX_NAME); } + public void testRestoreFlowWithForceEmptyTranslogNoOp() throws Exception { + prepareCluster(1, 3, INDEX_NAME, 0, 1); + Map indexStats = indexData(randomIntBetween(2, 3), randomBoolean(), INDEX_NAME); + + assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), indexStats.get(REFRESHED_OR_FLUSHED_OPERATIONS)); + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); + ensureRed(INDEX_NAME); + + // Do not delete data from remote translog. If data is present in remote translog, forceEmptyTranslog is ignored + + logger.info("--> Restore with forceEmptyTranslog, should have no effect as remote translog has data"); + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); + client().admin() + .cluster() + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(INDEX_NAME).restoreAllShards(true).forceEmptyTranslog(true), + PlainActionFuture.newFuture() + ); + ensureGreen(INDEX_NAME); + + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + refresh(INDEX_NAME); + assertBusy( + () -> assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), indexStats.get(TOTAL_OPERATIONS)), + 30, + TimeUnit.SECONDS + ); + } + + + public void testRestoreFlowWithForceEmptyTranslog() throws Exception { + prepareCluster(1, 3, INDEX_NAME, 0, 1); + Map indexStats = indexData(randomIntBetween(2, 3), randomBoolean(), INDEX_NAME); + + assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), indexStats.get(REFRESHED_OR_FLUSHED_OPERATIONS)); + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primaryNodeName(INDEX_NAME))); + ensureRed(INDEX_NAME); + + // Delete data from remote translog so that forceEmptyTranslog can take effect + String indexUUID = client().admin() + .indices() + .prepareGetSettings(INDEX_NAME) + .get() + .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID); + + Path remoteTranslogMetadataPath = Path.of(String.valueOf(translogRepoPath), indexUUID, "/0/translog/metadata"); + Path remoteTranslogDataPath = Path.of(String.valueOf(translogRepoPath), indexUUID, "/0/translog/data"); + IOUtils.rm(remoteTranslogDataPath, remoteTranslogMetadataPath); + + logger.info("--> Restore with forceEmptyTranslog, should turn the index green"); + assertAcked(client().admin().indices().prepareClose(INDEX_NAME)); + client().admin() + .cluster() + .restoreRemoteStore( + new RestoreRemoteStoreRequest().indices(INDEX_NAME).restoreAllShards(true).forceEmptyTranslog(true), + PlainActionFuture.newFuture() + ); + ensureGreen(INDEX_NAME); + + ensureYellowAndNoInitializingShards(INDEX_NAME); + ensureGreen(INDEX_NAME); + refresh(INDEX_NAME); + assertBusy( + () -> assertHitCount(client().prepareSearch(INDEX_NAME).setSize(0).get(), indexStats.get(REFRESHED_OR_FLUSHED_OPERATIONS)), + 30, + TimeUnit.SECONDS + ); + } + /** * Simulates refreshed data restored using Remote Segment Store * and unrefreshed data restored using Remote Translog Store diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index 6ff7a2449dd70..7d0cbaf9986fa 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -407,7 +407,7 @@ void recoverFromSnapshotAndRemoteStore( if (indexShard.indexSettings.isRemoteTranslogStoreEnabled() == false) { bootstrap(indexShard, store); } else { - bootstrapForSnapshot(indexShard, store); + bootstrapFromLastCommit(indexShard, store); } assert indexShard.shardRouting.primary() : "only primary shards can recover from store"; writeEmptyRetentionLeasesFile(indexShard); @@ -555,7 +555,7 @@ private void recoverFromRemoteStore(IndexShard indexShard) throws IndexShardReco } } else if (remoteSegmentEmpty == false && remoteTranslogEmpty) { if (((RecoverySource.RemoteStoreRecoverySource) indexShard.shardRouting.recoverySource()).forceEmptyTranslog()) { - bootstrap(indexShard, store); + bootstrapFromLastCommit(indexShard, store); } } @@ -695,7 +695,7 @@ private void restore( if (indexShard.indexSettings.isRemoteTranslogStoreEnabled() == false) { bootstrap(indexShard, store); } else { - bootstrapForSnapshot(indexShard, store); + bootstrapFromLastCommit(indexShard, store); } assert indexShard.shardRouting.primary() : "only primary shards can recover from store"; writeEmptyRetentionLeasesFile(indexShard); @@ -747,7 +747,7 @@ private void restore( } } - private void bootstrapForSnapshot(final IndexShard indexShard, final Store store) throws IOException { + private void bootstrapFromLastCommit(final IndexShard indexShard, final Store store) throws IOException { store.bootstrapNewHistory(); final SegmentInfos segmentInfos = store.readLastCommittedSegmentsInfo(); final long localCheckpoint = Long.parseLong(segmentInfos.userData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)); diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java index 414c82b4a470f..73dfa4ba0a656 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRestoreRemoteStoreAction.java @@ -45,6 +45,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC ); restoreRemoteStoreRequest.waitForCompletion(request.paramAsBoolean("wait_for_completion", false)); restoreRemoteStoreRequest.restoreAllShards(request.paramAsBoolean("restore_all_shards", false)); + restoreRemoteStoreRequest.forceEmptyTranslog(request.paramAsBoolean("force_empty_translog", false)); request.applyContentParser(p -> restoreRemoteStoreRequest.source(p.mapOrdered())); return channel -> client.admin().cluster().restoreRemoteStore(restoreRemoteStoreRequest, new RestToXContentListener<>(channel)); }