diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java index 072e03e8a2f79..37f43bb8ff58f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java @@ -157,6 +157,11 @@ public void testCloneSnapshotIndex() throws Exception { assertEquals(status1.getStats().getTotalSize(), status2.getStats().getTotalSize()); } + /** + * This test checks for lock file count in remote store after cloning the snapshot and also validates that + * shard generation doesn't change after taking a shallow clone snapshot. + * @throws Exception + */ public void testCloneShallowSnapshotIndex() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -168,16 +173,9 @@ public void testCloneShallowSnapshotIndex() throws Exception { final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "fs", snapshotRepoPath); - final String shallowSnapshotRepoName = "shallow-snapshot-repo-name"; - final Path shallowSnapshotRepoPath = randomRepoPath(); - createRepository(shallowSnapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath)); - final Path remoteStoreRepoPath = randomRepoPath(); createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; - createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); - final String remoteStoreEnabledIndexName = "remote-index-1"; final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); @@ -187,21 +185,145 @@ public void testCloneShallowSnapshotIndex() throws Exception { createFullSnapshot(snapshotRepoName, snapshot); assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0); - indexRandomDocs(indexName, randomIntBetween(20, 100)); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(20, 100)); + logger.info("Updating repo setting to enable shallow copy snapshots"); + createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); final String shallowSnapshot = "shallow-snapshot"; - createFullSnapshot(shallowSnapshotRepoName, shallowSnapshot); + createFullSnapshot(snapshotRepoName, shallowSnapshot); assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); - if (randomBoolean()) { - assertAcked(admin().indices().prepareDelete(indexName)); - } + final RepositoryData repositoryData = getRepositoryData(snapshotRepoName); + final IndexId indexId = repositoryData.resolveIndexId(remoteStoreEnabledIndexName); + final int shardId = 0; + final String currentShardGen = repositoryData.shardGenerations().getShardGen(indexId, shardId); final String sourceSnapshot = shallowSnapshot; final String targetSnapshot = "target-snapshot"; + assertAcked(startClone(snapshotRepoName, sourceSnapshot, targetSnapshot, remoteStoreEnabledIndexName).get()); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 2); + + final RepositoryData updatedRepositoryData = getRepositoryData(snapshotRepoName); + final String updatedShardGen = updatedRepositoryData.shardGenerations().getShardGen(indexId, shardId); + assert (updatedShardGen.equals(currentShardGen)); + } + + /** + * This test checks that cloning a partial shallow copy snapshot fails. + * @throws Exception + */ + public void testShallowClonePartialSourceSnapshot() throws Exception { + disableRepoConsistencyCheck("Remote store repository is being used in the test"); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + + internalCluster().startClusterManagerOnlyNode(); + final String dataNode = internalCluster().startDataOnlyNode(); + ensureStableCluster(2); + final String clusterManagerNode = internalCluster().getClusterManagerName(); + + final String snapshotRepoName = "snapshot-repo-name"; + final Path snapshotRepoPath = randomRepoPath(); + createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); + + final Path remoteStoreRepoPath = randomRepoPath(); + createRepository("remote-store-repo-name", "fs", remoteStoreRepoPath); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); + + // Creating a partial shallow copy snapshot + final String snapshot = "snapshot"; + blockNodeWithIndex(snapshotRepoName, remoteStoreEnabledIndexName); + blockDataNode(snapshotRepoName, dataNode); + + final Client clusterManagerClient = internalCluster().clusterManagerClient(); + final ActionFuture snapshotFuture = clusterManagerClient.admin() + .cluster() + .prepareCreateSnapshot(snapshotRepoName, snapshot) + .setWaitForCompletion(true) + .execute(); + + awaitNumberOfSnapshotsInProgress(1); + waitForBlock(dataNode, snapshotRepoName, TimeValue.timeValueSeconds(30L)); + internalCluster().restartNode(dataNode); + assertThat(snapshotFuture.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL)); + + unblockAllDataNodes(snapshotRepoName); + ensureStableCluster(2, clusterManagerNode); + + final SnapshotException sne = expectThrows( + SnapshotException.class, + () -> startClone(clusterManagerClient, snapshotRepoName, snapshot, "target-snapshot", remoteStoreEnabledIndexName).actionGet( + TimeValue.timeValueSeconds(30L) + ) + ); + assertThat( + sne.getMessage(), + containsString( + "Can't clone index [" + getRepositoryData(snapshotRepoName).resolveIndexId(remoteStoreEnabledIndexName) + "] because its snapshot was not successful." + ) + ); + } + + /** + * This test checks that clone operation fails if there's a failure while cloning the lock file. + * @throws Exception + */ + public void testShallowCloneAcquireLockFailed() throws Exception { + disableRepoConsistencyCheck("This test uses remote store repository"); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings("remote-store-repo-name")); + internalCluster().startDataOnlyNode(); + + logger.info("--> creating snapshot repository"); + final String shallowSnapshotRepoName = "shallow-snapshot-repo-name"; + final Path shallowSnapshotRepoPath = randomRepoPath(); + createRepository(shallowSnapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath)); + + logger.info("--> creating remote store repository"); + final String remoteStoreRepoName = "remote-store-repo-name"; + final Path remoteStoreRepoPath = randomRepoPath(); + Settings.Builder remoteStoreRepoSettingsBuilder = Settings.builder() + .put("location", remoteStoreRepoPath); + createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder); + + final String indexName = "index-1"; + createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); + + final String snapshot = "snapshot"; + createFullSnapshot(shallowSnapshotRepoName, snapshot); + + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); + assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 1); + + logger.info("Updating repo settings"); + remoteStoreRepoSettingsBuilder.putList("regexes_to_fail_io", "lock$"); + createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder); + + final String sourceSnapshot = snapshot; + final String targetSnapshot = "target-snapshot"; + expectThrows( + ExecutionException.class, + () -> startClone(shallowSnapshotRepoName, sourceSnapshot, targetSnapshot, indexName, remoteStoreEnabledIndexName).get() + ); + + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); + assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 1); + + logger.info("Updating repo settings"); + remoteStoreRepoSettingsBuilder.remove("regexes_to_fail_io"); + createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder); + assertAcked(startClone(shallowSnapshotRepoName, sourceSnapshot, targetSnapshot, indexName, remoteStoreEnabledIndexName).get()); - logger.info("Lock files count: {}", getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length); assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 2); + assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 2); } public void testShallowCloneNameAvailability() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java index 3a9eacd6ac183..f28db006577ae 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -137,6 +137,7 @@ public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception { .prepareDeleteSnapshot(snapshotRepoName, snapshotsToBeDeleted.toArray(new String[0])) .get() ); + awaitNoMoreRunningOperations(); assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalShallowCopySnapshotsCount - tobeDeletedSnapshotsCount); assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == totalShallowCopySnapshotsCount - tobeDeletedSnapshotsCount); @@ -296,6 +297,65 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception { assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == 0); } + // Checking snapshot deletion after inducing lock release failure for shallow copy snapshot. + public void testReleaseLockFailure() throws Exception { + disableRepoConsistencyCheck("This test uses remote store repository"); + FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings("remote-store-repo-name")); + internalCluster().startDataOnlyNode(); + + logger.info("--> creating snapshot repository"); + final String shallowSnapshotRepoName = "shallow-snapshot-repo-name"; + final Path shallowSnapshotRepoPath = randomRepoPath(); + createRepository(shallowSnapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath)); + + logger.info("--> creating remote store repository"); + final String remoteStoreRepoName = "remote-store-repo-name"; + final Path remoteStoreRepoPath = randomRepoPath(); + Settings.Builder remoteStoreRepoSettingsBuilder = Settings.builder() + .put("location", remoteStoreRepoPath); + createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder); + + createIndexWithContent("index-test-1"); + + final String remoteStoreEnabledIndexName = "remote-index-1"; + final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); + createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); + indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10)); + + createFullSnapshot(shallowSnapshotRepoName, "snapshot_one"); + + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); + assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 1); + + logger.info("Updating repo settings"); + remoteStoreRepoSettingsBuilder.putList("regexes_to_fail_io", "lock$"); + createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder); + + // Deleting snapshot_one after updating repo settings to fail on lock file release operation. + // Expecting snapshot to be deleted but the lock file as well as snapshot files to be still present. + // index.N and index.latest are the only two files expected to be present after snapshot deletion. + assertAcked(startDeleteSnapshot(shallowSnapshotRepoName, "snapshot_one").get()); + assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 0); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1); + assert(numberOfFiles(shallowSnapshotRepoPath) != 2); + + logger.info("Updating repo settings"); + remoteStoreRepoSettingsBuilder.remove("regexes_to_fail_io"); + createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder); + + createIndexWithContent("test-index-2"); + createFullSnapshot(shallowSnapshotRepoName, "snapshot_two"); + + // Deleting snapshot_two after reverting the repo settings. Expecting snapshot count as well as lock + // file count to be zero after the snapshot deletion. Also the files present in the snapshot repository + // should equal to 2 (index.latest and index.N) + assertAcked(startDeleteSnapshot(shallowSnapshotRepoName, "snapshot_two").get()); + assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 0); + assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0); + assert(numberOfFiles(shallowSnapshotRepoPath) == 2); + } + public void testRemoteStoreCleanupForDeletedIndex() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java index fb91b1d7a006c..718a9ee9cda5f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java @@ -40,6 +40,7 @@ import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.FeatureFlagSetter; import org.opensearch.threadpool.ThreadPool; import java.nio.file.Path; @@ -99,6 +100,8 @@ public void testStatusAPICallForShallowCopySnapshot() throws Exception { assertThat(shallowSnapshotShardState.getStats().getTotalSize(), greaterThan(0L)); assertThat(shallowSnapshotShardState.getStats().getIncrementalFileCount(), is(0)); assertThat(shallowSnapshotShardState.getStats().getIncrementalSize(), is(0L)); + assertNotNull(shallowSnapshotShardState.getStats().getStartTime()); + assertNotNull(shallowSnapshotShardState.getStats().getTime()); } public void testStatusAPIStatsForBackToBackShallowSnapshot() throws Exception { diff --git a/test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java b/test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java index 7db71c4be0968..9817ab0a0adde 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java @@ -74,6 +74,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; import java.util.stream.Collectors; public class MockRepository extends FsRepository { @@ -125,6 +126,8 @@ public long getFailureCount() { private final List skipExceptionOnBlobs; + private final List regexesToFailIO; + private final boolean useLuceneCorruptionException; private final long maximumNumberOfFailures; @@ -183,6 +186,7 @@ public MockRepository( super(overrideSettings(metadata, environment), environment, namedXContentRegistry, clusterService, recoverySettings); randomControlIOExceptionRate = metadata.settings().getAsDouble("random_control_io_exception_rate", 0.0); randomDataFileIOExceptionRate = metadata.settings().getAsDouble("random_data_file_io_exception_rate", 0.0); + regexesToFailIO = metadata.settings().getAsList("regexes_to_fail_io"); skipExceptionOnVerificationFile = metadata.settings().getAsBoolean("skip_exception_on_verification_file", false); skipExceptionOnListBlobs = metadata.settings().getAsBoolean("skip_exception_on_list_blobs", false); skipExceptionOnBlobs = metadata.settings().getAsList("skip_exception_on_blobs"); @@ -388,6 +392,12 @@ private void maybeIOExceptionOrBlock(String blobName) throws IOException { // Condition 4 - This condition allows to skip exception on specific blobName or blobPrefix return; } + + if (failIOForBlobsMatchingRegex(blobName) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) { + logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path()); + throw new IOException("Random IOException"); + } + if (blobName.startsWith("__")) { if (shouldFail(blobName, randomDataFileIOExceptionRate) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) { logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path()); @@ -497,6 +507,9 @@ public void deleteBlobsIgnoringIfNotExists(List blobNames) throws IOExce if (blockOnDeleteIndexN && blobNames.stream().anyMatch(name -> name.startsWith(BlobStoreRepository.INDEX_FILE_PREFIX))) { blockExecutionAndMaybeWait("index-{N}"); } + if(blobNames.stream().anyMatch(blobName -> failIOForBlobsMatchingRegex(blobName))) { + throw new IOException("Random Exception"); + } if (setThrowExceptionWhileDelete) { throw new IOException("Random exception"); } @@ -597,4 +610,8 @@ private boolean skipExceptionOnBlob(String blobName) { return skipExceptionOnBlobs.contains(blobName); } } + + private boolean failIOForBlobsMatchingRegex(String blobName) { + return regexesToFailIO.stream().anyMatch(regex -> Pattern.compile(regex).matcher(blobName).find()); + } }