Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Snapshot Interop] Adding more ITs for Snapshot Interoperability with… #9785

Closed
wants to merge 1 commit into from
Closed

[Snapshot Interop] Adding more ITs for Snapshot Interoperability with… #9785

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 136 additions & 13 deletions server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,11 @@ public void testCloneSnapshotIndex() throws Exception {
assertEquals(status1.getStats().getTotalSize(), status2.getStats().getTotalSize());
}

/**
* This test checks for lock file count in remote store after cloning the snapshot and also validates that
* shard generation doesn't change after taking a shallow clone snapshot.
* @throws Exception
*/
public void testCloneShallowSnapshotIndex() throws Exception {
disableRepoConsistencyCheck("This test uses remote store repository");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
Expand All @@ -168,16 +173,9 @@ public void testCloneShallowSnapshotIndex() throws Exception {
final Path snapshotRepoPath = randomRepoPath();
createRepository(snapshotRepoName, "fs", snapshotRepoPath);

final String shallowSnapshotRepoName = "shallow-snapshot-repo-name";
final Path shallowSnapshotRepoPath = randomRepoPath();
createRepository(shallowSnapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath));

final Path remoteStoreRepoPath = randomRepoPath();
createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath);

final String indexName = "index-1";
createIndexWithRandomDocs(indexName, randomIntBetween(5, 10));

final String remoteStoreEnabledIndexName = "remote-index-1";
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings();
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
Expand All @@ -187,21 +185,146 @@ public void testCloneShallowSnapshotIndex() throws Exception {
createFullSnapshot(snapshotRepoName, snapshot);
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0);

indexRandomDocs(indexName, randomIntBetween(20, 100));
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(20, 100));

logger.info("Updating repo setting to enable shallow copy snapshots");
createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(snapshotRepoPath));
final String shallowSnapshot = "shallow-snapshot";
createFullSnapshot(shallowSnapshotRepoName, shallowSnapshot);
createFullSnapshot(snapshotRepoName, shallowSnapshot);
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1);

if (randomBoolean()) {
assertAcked(admin().indices().prepareDelete(indexName));
}
final RepositoryData repositoryData = getRepositoryData(snapshotRepoName);
final IndexId indexId = repositoryData.resolveIndexId(remoteStoreEnabledIndexName);
final int shardId = 0;
final String currentShardGen = repositoryData.shardGenerations().getShardGen(indexId, shardId);

final String sourceSnapshot = shallowSnapshot;
final String targetSnapshot = "target-snapshot";
assertAcked(startClone(snapshotRepoName, sourceSnapshot, targetSnapshot, remoteStoreEnabledIndexName).get());
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 2);

final RepositoryData updatedRepositoryData = getRepositoryData(snapshotRepoName);
final String updatedShardGen = updatedRepositoryData.shardGenerations().getShardGen(indexId, shardId);
assert (updatedShardGen.equals(currentShardGen));
}

/**
* This test checks that cloning a partial shallow copy snapshot fails.
* @throws Exception
*/
public void testShallowClonePartialSourceSnapshot() throws Exception {
disableRepoConsistencyCheck("Remote store repository is being used in the test");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);

internalCluster().startClusterManagerOnlyNode();
final String dataNode = internalCluster().startDataOnlyNode();
ensureStableCluster(2);
final String clusterManagerNode = internalCluster().getClusterManagerName();

final String snapshotRepoName = "snapshot-repo-name";
final Path snapshotRepoPath = randomRepoPath();
createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath));

final Path remoteStoreRepoPath = randomRepoPath();
createRepository("remote-store-repo-name", "fs", remoteStoreRepoPath);

final String remoteStoreEnabledIndexName = "remote-index-1";
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings();
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10));
Comment on lines +217 to +234
Copy link
Contributor

@harishbhakuni harishbhakuni Oct 3, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you move these common lines to a different method, so that they can be reused in other tests?


// Creating a partial shallow copy snapshot
final String snapshot = "snapshot";
blockNodeWithIndex(snapshotRepoName, remoteStoreEnabledIndexName);
blockDataNode(snapshotRepoName, dataNode);

final Client clusterManagerClient = internalCluster().clusterManagerClient();
final ActionFuture<CreateSnapshotResponse> snapshotFuture = clusterManagerClient.admin()
.cluster()
.prepareCreateSnapshot(snapshotRepoName, snapshot)
.setWaitForCompletion(true)
.execute();

awaitNumberOfSnapshotsInProgress(1);
waitForBlock(dataNode, snapshotRepoName, TimeValue.timeValueSeconds(30L));
internalCluster().restartNode(dataNode);
assertThat(snapshotFuture.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL));

unblockAllDataNodes(snapshotRepoName);
ensureStableCluster(2, clusterManagerNode);

final SnapshotException sne = expectThrows(
SnapshotException.class,
() -> startClone(clusterManagerClient, snapshotRepoName, snapshot, "target-snapshot", remoteStoreEnabledIndexName).actionGet(
TimeValue.timeValueSeconds(30L)
)
);
assertThat(
sne.getMessage(),
containsString(
"Can't clone index ["
+ getRepositoryData(snapshotRepoName).resolveIndexId(remoteStoreEnabledIndexName)
+ "] because its snapshot was not successful."
)
);
}

/**
* This test checks that clone operation fails if there's a failure while cloning the lock file.
* @throws Exception
*/
public void testShallowCloneAcquireLockFailed() throws Exception {
disableRepoConsistencyCheck("This test uses remote store repository");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we don't need this flag now as REMOTE_STORE is GA since 2.10.

internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings("remote-store-repo-name"));
internalCluster().startDataOnlyNode();

logger.info("--> creating snapshot repository");
final String shallowSnapshotRepoName = "shallow-snapshot-repo-name";
final Path shallowSnapshotRepoPath = randomRepoPath();
createRepository(shallowSnapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath));

logger.info("--> creating remote store repository");
final String remoteStoreRepoName = "remote-store-repo-name";
final Path remoteStoreRepoPath = randomRepoPath();
Settings.Builder remoteStoreRepoSettingsBuilder = Settings.builder().put("location", remoteStoreRepoPath);
createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder);

final String indexName = "index-1";
createIndexWithRandomDocs(indexName, randomIntBetween(5, 10));

final String remoteStoreEnabledIndexName = "remote-index-1";
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings();
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10));

final String snapshot = "snapshot";
createFullSnapshot(shallowSnapshotRepoName, snapshot);

assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1);
assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 1);

logger.info("Updating repo settings");
remoteStoreRepoSettingsBuilder.putList("regexes_to_fail_io", "lock$");
createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder);

final String sourceSnapshot = snapshot;
final String targetSnapshot = "target-snapshot";
expectThrows(
ExecutionException.class,
() -> startClone(shallowSnapshotRepoName, sourceSnapshot, targetSnapshot, indexName, remoteStoreEnabledIndexName).get()
);

assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1);
assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 1);

logger.info("Updating repo settings");
remoteStoreRepoSettingsBuilder.remove("regexes_to_fail_io");
createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder);

assertAcked(startClone(shallowSnapshotRepoName, sourceSnapshot, targetSnapshot, indexName, remoteStoreEnabledIndexName).get());
logger.info("Lock files count: {}", getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length);
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 2);
assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 2);
}

public void testShallowCloneNameAvailability() throws Exception {
Expand Down
59 changes: 59 additions & 0 deletions server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception {
.prepareDeleteSnapshot(snapshotRepoName, snapshotsToBeDeleted.toArray(new String[0]))
.get()
);
awaitNoMoreRunningOperations();
assert (getRepositoryData(snapshotRepoName).getSnapshotIds().size() == totalShallowCopySnapshotsCount - tobeDeletedSnapshotsCount);
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == totalShallowCopySnapshotsCount
- tobeDeletedSnapshotsCount);
Expand Down Expand Up @@ -296,6 +297,64 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception {
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME).length == 0);
}

// Checking snapshot deletion after inducing lock release failure for shallow copy snapshot.
public void testReleaseLockFailure() throws Exception {
disableRepoConsistencyCheck("This test uses remote store repository");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings("remote-store-repo-name"));
internalCluster().startDataOnlyNode();

logger.info("--> creating snapshot repository");
final String shallowSnapshotRepoName = "shallow-snapshot-repo-name";
final Path shallowSnapshotRepoPath = randomRepoPath();
createRepository(shallowSnapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath));

logger.info("--> creating remote store repository");
final String remoteStoreRepoName = "remote-store-repo-name";
final Path remoteStoreRepoPath = randomRepoPath();
Settings.Builder remoteStoreRepoSettingsBuilder = Settings.builder().put("location", remoteStoreRepoPath);
createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder);

createIndexWithContent("index-test-1");

final String remoteStoreEnabledIndexName = "remote-index-1";
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings();
createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings);
indexRandomDocs(remoteStoreEnabledIndexName, randomIntBetween(5, 10));

createFullSnapshot(shallowSnapshotRepoName, "snapshot_one");

assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1);
assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 1);

logger.info("Updating repo settings");
remoteStoreRepoSettingsBuilder.putList("regexes_to_fail_io", "lock$");
createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder);

// Deleting snapshot_one after updating repo settings to fail on lock file release operation.
// Expecting snapshot to be deleted but the lock file as well as snapshot files to be still present.
// index.N and index.latest are the only two files expected to be present after snapshot deletion.
assertAcked(startDeleteSnapshot(shallowSnapshotRepoName, "snapshot_one").get());
assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 0);
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 1);
assert (numberOfFiles(shallowSnapshotRepoPath) != 2);

logger.info("Updating repo settings");
remoteStoreRepoSettingsBuilder.remove("regexes_to_fail_io");
createRepository(remoteStoreRepoName, "mock", remoteStoreRepoSettingsBuilder);

createIndexWithContent("test-index-2");
createFullSnapshot(shallowSnapshotRepoName, "snapshot_two");

// Deleting snapshot_two after reverting the repo settings. Expecting snapshot count as well as lock
// file count to be zero after the snapshot deletion. Also the files present in the snapshot repository
// should equal to 2 (index.latest and index.N)
assertAcked(startDeleteSnapshot(shallowSnapshotRepoName, "snapshot_two").get());
assert (clusterAdmin().prepareGetSnapshots(shallowSnapshotRepoName).get().getSnapshots().size() == 0);
assert (getLockFilesInRemoteStore(remoteStoreEnabledIndexName, remoteStoreRepoName).length == 0);
assert (numberOfFiles(shallowSnapshotRepoPath) == 2);
}

public void testRemoteStoreCleanupForDeletedIndex() throws Exception {
disableRepoConsistencyCheck("Remote store repository is being used in the test");
FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE);
Expand Down
2 changes: 2 additions & 0 deletions ...src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ public void testStatusAPICallForShallowCopySnapshot() throws Exception {
assertThat(shallowSnapshotShardState.getStats().getTotalSize(), greaterThan(0L));
assertThat(shallowSnapshotShardState.getStats().getIncrementalFileCount(), is(0));
assertThat(shallowSnapshotShardState.getStats().getIncrementalSize(), is(0L));
assertNotNull(shallowSnapshotShardState.getStats().getStartTime());
assertNotNull(shallowSnapshotShardState.getStats().getTime());
}

public void testStatusAPIStatsForBackToBackShallowSnapshot() throws Exception {
Expand Down
17 changes: 17 additions & 0 deletions test/framework/src/main/java/org/opensearch/snapshots/mockstore/MockRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

public class MockRepository extends FsRepository {
Expand Down Expand Up @@ -125,6 +126,8 @@ public long getFailureCount() {

private final List<String> skipExceptionOnBlobs;

private final List<String> regexesToFailIO;

private final boolean useLuceneCorruptionException;

private final long maximumNumberOfFailures;
Expand Down Expand Up @@ -183,6 +186,7 @@ public MockRepository(
super(overrideSettings(metadata, environment), environment, namedXContentRegistry, clusterService, recoverySettings);
randomControlIOExceptionRate = metadata.settings().getAsDouble("random_control_io_exception_rate", 0.0);
randomDataFileIOExceptionRate = metadata.settings().getAsDouble("random_data_file_io_exception_rate", 0.0);
regexesToFailIO = metadata.settings().getAsList("regexes_to_fail_io");
skipExceptionOnVerificationFile = metadata.settings().getAsBoolean("skip_exception_on_verification_file", false);
skipExceptionOnListBlobs = metadata.settings().getAsBoolean("skip_exception_on_list_blobs", false);
skipExceptionOnBlobs = metadata.settings().getAsList("skip_exception_on_blobs");
Expand Down Expand Up @@ -388,6 +392,12 @@ private void maybeIOExceptionOrBlock(String blobName) throws IOException {
// Condition 4 - This condition allows to skip exception on specific blobName or blobPrefix
return;
}

if (failIOForBlobsMatchingRegex(blobName) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) {
logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path());
throw new IOException("Random IOException");
}

if (blobName.startsWith("__")) {
if (shouldFail(blobName, randomDataFileIOExceptionRate) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) {
logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path());
Expand Down Expand Up @@ -497,6 +507,9 @@ public void deleteBlobsIgnoringIfNotExists(List<String> blobNames) throws IOExce
if (blockOnDeleteIndexN && blobNames.stream().anyMatch(name -> name.startsWith(BlobStoreRepository.INDEX_FILE_PREFIX))) {
blockExecutionAndMaybeWait("index-{N}");
}
if (blobNames.stream().anyMatch(blobName -> failIOForBlobsMatchingRegex(blobName))) {
throw new IOException("Random Exception");
}
if (setThrowExceptionWhileDelete) {
throw new IOException("Random exception");
}
Expand Down Expand Up @@ -597,4 +610,8 @@ private boolean skipExceptionOnBlob(String blobName) {
return skipExceptionOnBlobs.contains(blobName);
}
}

private boolean failIOForBlobsMatchingRegex(String blobName) {
return regexesToFailIO.stream().anyMatch(regex -> Pattern.compile(regex).matcher(blobName).find());
}
}
Loading