Skip to content

Commit

Permalink
[Snapshot Interop] Adding more ITs for Snapshot Interoperability with…
Browse files Browse the repository at this point in the history 
… Remote Store.

Signed-off-by: Bansi Kasundra <[email protected]>
  • Loading branch information
@kasundra07
kasundra07 committed Sep 6, 2023
1 parent 9d6c43a commit fe56474
Show file tree
Hide file tree
Showing 4 changed files with 215 additions and 13 deletions.
148 changes: 135 additions & 13 deletions server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,11 @@ public void testCloneSnapshotIndex() throws Exception {
assertEquals(status1.getStats().getTotalSize(), status2.getStats().getTotalSize());
}

/**
* This test checks for lock file count in remote store after cloning the snapshot and also validates that
* shard generation doesn't change after taking a shallow clone snapshot.
* @throws Exception
*/
public void testCloneShallowSnapshotIndex() throws Exception {
disableRepoConsistencyCheck("This test uses remote store repository");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
Expand All @@ -168,16 +173,9 @@ public void testCloneShallowSnapshotIndex() throws Exception {
final Path snapshotRepoPath = randomRepoPath();
createRepository(snapshotRepoName, "fs", snapshotRepoPath);

final String shallowSnapshotRepoName = "shallow-snapshot-repo-name";
final Path shallowSnapshotRepoPath = randomRepoPath();
createRepository(shallowSnapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath));

final Path remoteStoreRepoPath = randomRepoPath();
createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath);

final String indexName = "index-1";
createIndexWithRandomDocs(indexName, randomIntBetween(5, 10));

final String remoteStoreEnabledIndexName = "remote-index-1";
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings();
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
Expand All @@ -187,21 +185,145 @@ public void testCloneShallowSnapshotIndex() throws Exception {
createFullSnapshot(snapshotRepoName, snapshot);
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0);

indexRandomDocs(indexName, randomIntBetween(20, 100));
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(20, 100));

logger.info("Updating repo setting to enable shallow copy snapshots");
createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(snapshotRepoPath));
final String shallowSnapshot = "shallow-snapshot";
createFullSnapshot(shallowSnapshotRepoName, shallowSnapshot);
createFullSnapshot(snapshotRepoName, shallowSnapshot);
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1);

if (randomBoolean()) {
assertAcked(admin().indices().prepareDelete(indexName));
}
final RepositoryData repositoryData = getRepositoryData(snapshotRepoName);
final IndexId indexId = repositoryData.resolveIndexId(remoteStoreEnabledIndexName);
final int shardId = 0;
final String currentShardGen = repositoryData.shardGenerations().getShardGen(indexId, shardId);

final String sourceSnapshot = shallowSnapshot;
final String targetSnapshot = "target-snapshot";
assertAcked(startClone(snapshotRepoName, sourceSnapshot, targetSnapshot, remoteStoreEnabledIndexName).get());
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 2);

final RepositoryData updatedRepositoryData = getRepositoryData(snapshotRepoName);
final String updatedShardGen = updatedRepositoryData.shardGenerations().getShardGen(indexId, shardId);
assert (updatedShardGen.equals(currentShardGen));
}

/**
* This test checks that cloning a partial shallow copy snapshot fails.
* @throws Exception
*/
public void testShallowClonePartialSourceSnapshot() throws Exception {
disableRepoConsistencyCheck("Remote store repository is being used in the test");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);

internalCluster().startClusterManagerOnlyNode();
final String dataNode = internalCluster().startDataOnlyNode();
ensureStableCluster(2);
final String clusterManagerNode = internalCluster().getClusterManagerName();

final String snapshotRepoName = "snapshot-repo-name";
final Path snapshotRepoPath = randomRepoPath();
createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath));

final Path remoteStoreRepoPath = randomRepoPath();
createRepository("remote-store-repo-name", "fs", remoteStoreRepoPath);

final String remoteStoreEnabledIndexName = "remote-index-1";
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings();
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10));

// Creating a partial shallow copy snapshot
final String snapshot = "snapshot";
blockNodeWithIndex(snapshotRepoName, remoteStoreEnabledIndexName);
blockDataNode(snapshotRepoName, dataNode);

final Client clusterManagerClient = internalCluster().clusterManagerClient();
final ActionFuture<CreateSnapshotResponse> snapshotFuture = clusterManagerClient.admin()
.cluster()
.prepareCreateSnapshot(snapshotRepoName, snapshot)
.setWaitForCompletion(true)
.execute();

awaitNumberOfSnapshotsInProgress(1);
waitForBlock(dataNode, snapshotRepoName, TimeValue.timeValueSeconds(30L));
internalCluster().restartNode(dataNode);
assertThat(snapshotFuture.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL));

unblockAllDataNodes(snapshotRepoName);
ensureStableCluster(2, clusterManagerNode);

final SnapshotException sne = expectThrows(
SnapshotException.class,
() -> startClone(clusterManagerClient, snapshotRepoName, snapshot, "target-snapshot", remoteStoreEnabledIndexName).actionGet(
TimeValue.timeValueSeconds(30L)
)
);
assertThat(
sne.getMessage(),
containsString(
"Can't clone index [" + getRepositoryData(snapshotRepoName).resolveIndexId(remoteStoreEnabledIndexName) + "] because its snapshot was not successful."
)
);
}

/**
* This test checks that clone operation fails if there's a failure while cloning the lock file.
* @throws Exception
*/
public void testShallowCloneAcquireLockFailed() throws Exception {
disableRepoConsistencyCheck("This test uses remote store repository");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings("remote-store-repo-name"));
internalCluster().startDataOnlyNode();

logger.info("--> creating snapshot repository");
final String shallowSnapshotRepoName = "shallow-snapshot-repo-name";
final Path shallowSnapshotRepoPath = randomRepoPath();
createRepository(shallowSnapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath));

logger.info("--> creating remote store repository");
final String remoteStoreRepoName = "remote-store-repo-name";
final Path remoteStoreRepoPath = randomRepoPath();
Settings.Builder remoteStoreRepoSettingsBuilder = Settings.builder()
.put("location", remoteStoreRepoPath);
createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder);

final String indexName = "index-1";
createIndexWithRandomDocs(indexName, randomIntBetween(5, 10));

final String remoteStoreEnabledIndexName = "remote-index-1";
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings();
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10));

final String snapshot = "snapshot";
createFullSnapshot(shallowSnapshotRepoName, snapshot);

assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1);
assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 1);

logger.info("Updating repo settings");
remoteStoreRepoSettingsBuilder.putList("regexes_to_fail_io", "lock$");
createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder);

final String sourceSnapshot = snapshot;
final String targetSnapshot = "target-snapshot";
expectThrows(
ExecutionException.class,
() -> startClone(shallowSnapshotRepoName, sourceSnapshot, targetSnapshot, indexName, remoteStoreEnabledIndexName).get()
);

assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1);
assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 1);

logger.info("Updating repo settings");
remoteStoreRepoSettingsBuilder.remove("regexes_to_fail_io");
createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder);

assertAcked(startClone(shallowSnapshotRepoName, sourceSnapshot, targetSnapshot, indexName, remoteStoreEnabledIndexName).get());
logger.info("Lock files count: {}", getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length);
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 2);
assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 2);
}

public void testShallowCloneNameAvailability() throws Exception {
Expand Down
60 changes: 60 additions & 0 deletions server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception {
.prepareDeleteSnapshot(snapshotRepoName, snapshotsToBeDeleted.toArray(new String[0]))
.get()
);
awaitNoMoreRunningOperations();
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalShallowCopySnapshotsCount - tobeDeletedSnapshotsCount);
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == totalShallowCopySnapshotsCount
- tobeDeletedSnapshotsCount);
Expand Down Expand Up @@ -296,6 +297,65 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception {
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == 0);
}

// Checking snapshot deletion after inducing lock release failure for shallow copy snapshot.
public void testReleaseLockFailure() throws Exception {
disableRepoConsistencyCheck("This test uses remote store repository");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings("remote-store-repo-name"));
internalCluster().startDataOnlyNode();

logger.info("--> creating snapshot repository");
final String shallowSnapshotRepoName = "shallow-snapshot-repo-name";
final Path shallowSnapshotRepoPath = randomRepoPath();
createRepository(shallowSnapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath));

logger.info("--> creating remote store repository");
final String remoteStoreRepoName = "remote-store-repo-name";
final Path remoteStoreRepoPath = randomRepoPath();
Settings.Builder remoteStoreRepoSettingsBuilder = Settings.builder()
.put("location", remoteStoreRepoPath);
createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder);

createIndexWithContent("index-test-1");

final String remoteStoreEnabledIndexName = "remote-index-1";
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings();
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10));

createFullSnapshot(shallowSnapshotRepoName, "snapshot_one");

assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1);
assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 1);

logger.info("Updating repo settings");
remoteStoreRepoSettingsBuilder.putList("regexes_to_fail_io", "lock$");
createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder);

// Deleting snapshot_one after updating repo settings to fail on lock file release operation.
// Expecting snapshot to be deleted but the lock file as well as snapshot files to be still present.
// index.N and index.latest are the only two files expected to be present after snapshot deletion.
assertAcked(startDeleteSnapshot(shallowSnapshotRepoName, "snapshot_one").get());
assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 0);
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1);
assert(numberOfFiles(shallowSnapshotRepoPath) != 2);

logger.info("Updating repo settings");
remoteStoreRepoSettingsBuilder.remove("regexes_to_fail_io");
createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder);

createIndexWithContent("test-index-2");
createFullSnapshot(shallowSnapshotRepoName, "snapshot_two");

// Deleting snapshot_two after reverting the repo settings. Expecting snapshot count as well as lock
// file count to be zero after the snapshot deletion. Also the files present in the snapshot repository
// should equal to 2 (index.latest and index.N)
assertAcked(startDeleteSnapshot(shallowSnapshotRepoName, "snapshot_two").get());
assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 0);
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0);
assert(numberOfFiles(shallowSnapshotRepoPath) == 2);
}

public void testRemoteStoreCleanupForDeletedIndex() throws Exception {
disableRepoConsistencyCheck("Remote store repository is being used in the test");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
Expand Down
3 changes: 3 additions & 0 deletions ...src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.opensearch.common.action.ActionFuture;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.test.FeatureFlagSetter;
import org.opensearch.threadpool.ThreadPool;

import java.nio.file.Path;
Expand Down Expand Up @@ -99,6 +100,8 @@ public void testStatusAPICallForShallowCopySnapshot() throws Exception {
assertThat(shallowSnapshotShardState.getStats().getTotalSize(), greaterThan(0L));
assertThat(shallowSnapshotShardState.getStats().getIncrementalFileCount(), is(0));
assertThat(shallowSnapshotShardState.getStats().getIncrementalSize(), is(0L));
assertNotNull(shallowSnapshotShardState.getStats().getStartTime());
assertNotNull(shallowSnapshotShardState.getStats().getTime());
}

public void testStatusAPIStatsForBackToBackShallowSnapshot() throws Exception {
Expand Down
17 changes: 17 additions & 0 deletions test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

public class MockRepository extends FsRepository {
Expand Down Expand Up @@ -125,6 +126,8 @@ public long getFailureCount() {

private final List<String> skipExceptionOnBlobs;

private final List<String> regexesToFailIO;

private final boolean useLuceneCorruptionException;

private final long maximumNumberOfFailures;
Expand Down Expand Up @@ -183,6 +186,7 @@ public MockRepository(
super(overrideSettings(metadata, environment), environment, namedXContentRegistry, clusterService, recoverySettings);
randomControlIOExceptionRate = metadata.settings().getAsDouble("random_control_io_exception_rate", 0.0);
randomDataFileIOExceptionRate = metadata.settings().getAsDouble("random_data_file_io_exception_rate", 0.0);
regexesToFailIO = metadata.settings().getAsList("regexes_to_fail_io");
skipExceptionOnVerificationFile = metadata.settings().getAsBoolean("skip_exception_on_verification_file", false);
skipExceptionOnListBlobs = metadata.settings().getAsBoolean("skip_exception_on_list_blobs", false);
skipExceptionOnBlobs = metadata.settings().getAsList("skip_exception_on_blobs");
Expand Down Expand Up @@ -388,6 +392,12 @@ private void maybeIOExceptionOrBlock(String blobName) throws IOException {
// Condition 4 - This condition allows to skip exception on specific blobName or blobPrefix
return;
}

if (failIOForBlobsMatchingRegex(blobName) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) {
logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path());
throw new IOException("Random IOException");
}

if (blobName.startsWith("__")) {
if (shouldFail(blobName, randomDataFileIOExceptionRate) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) {
logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path());
Expand Down Expand Up @@ -497,6 +507,9 @@ public void deleteBlobsIgnoringIfNotExists(List<String> blobNames) throws IOExce
if (blockOnDeleteIndexN && blobNames.stream().anyMatch(name -> name.startsWith(BlobStoreRepository.INDEX_FILE_PREFIX))) {
blockExecutionAndMaybeWait("index-{N}");
}
if(blobNames.stream().anyMatch(blobName -> failIOForBlobsMatchingRegex(blobName))) {
throw new IOException("Random Exception");
}
if (setThrowExceptionWhileDelete) {
throw new IOException("Random exception");
}
Expand Down Expand Up @@ -597,4 +610,8 @@ private boolean skipExceptionOnBlob(String blobName) {
return skipExceptionOnBlobs.contains(blobName);
}
}

private boolean failIOForBlobsMatchingRegex(String blobName) {
return regexesToFailIO.stream().anyMatch(regex -> Pattern.compile(regex).matcher(blobName).find());
}
}

0 comments on commit fe56474

Please sign in to comment.