Skip to content

Commit

Permalink
Take shallow copy snapshot when remote store migration is in progress (…
Browse files Browse the repository at this point in the history 
…#13309)

Signed-off-by: Lakshya Taragi <[email protected]>
  • Loading branch information
@ltaragi
ltaragi authored Apr 25, 2024
1 parent 42d46a9 commit 5bf522c
Show file tree
Hide file tree
Showing 4 changed files with 165 additions and 72 deletions.
81 changes: 9 additions & 72 deletions ...ClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,27 +8,15 @@

package org.opensearch.remotemigration;

import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.settings.SettingsException;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.index.IndexSettings;
import org.opensearch.indices.replication.common.ReplicationType;
import org.opensearch.snapshots.SnapshotInfo;
import org.opensearch.snapshots.SnapshotState;
import org.opensearch.test.InternalTestCluster;
import org.opensearch.test.OpenSearchIntegTestCase;

import java.nio.file.Path;
import java.util.Optional;

import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED;
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT;
import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.REMOTE_STORE;
Expand Down Expand Up @@ -92,13 +80,7 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix
assertNodeInCluster(remoteNodeName);

logger.info("Create a non remote-backed index");
client.admin()
.indices()
.prepareCreate(TEST_INDEX)
.setSettings(
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
)
.get();
createIndex(TEST_INDEX, 0);

logger.info("Verify that non remote stored backed index is created");
assertNonRemoteStoreBackedIndex(TEST_INDEX);
Expand All @@ -115,21 +97,12 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix

logger.info("Create snapshot of non remote stored backed index");

SnapshotInfo snapshotInfo = client().admin()
.cluster()
.prepareCreateSnapshot(snapshotRepoName, snapshotName)
.setIndices(TEST_INDEX)
.setWaitForCompletion(true)
.get()
.getSnapshotInfo();

assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
assertTrue(snapshotInfo.successfulShards() > 0);
assertEquals(0, snapshotInfo.failedShards());
createSnapshot(snapshotRepoName, snapshotName, TEST_INDEX);

logger.info("Restore index from snapshot under NONE direction");
String restoredIndexName1 = TEST_INDEX + "-restored1";
restoreSnapshot(snapshotRepoName, snapshotName, restoredIndexName1);
ensureGreen(restoredIndexName1);

logger.info("Verify that restored index is non remote-backed");
assertNonRemoteStoreBackedIndex(restoredIndexName1);
Expand All @@ -138,6 +111,7 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix
setDirection(REMOTE_STORE.direction);
String restoredIndexName2 = TEST_INDEX + "-restored2";
restoreSnapshot(snapshotRepoName, snapshotName, restoredIndexName2);
ensureGreen(restoredIndexName2);

logger.info("Verify that restored index is non remote-backed");
assertRemoteStoreBackedIndex(restoredIndexName2);
Expand All @@ -146,10 +120,10 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix
// compatibility mode setting test

public void testSwitchToStrictMode() throws Exception {
logger.info(" --> initialize cluster");
logger.info("Initialize cluster");
initializeCluster(false);

logger.info(" --> create a mixed mode cluster");
logger.info("Create a mixed mode cluster");
setClusterMode(MIXED.mode);
addRemote = true;
String remoteNodeName = internalCluster().startNode();
Expand All @@ -159,58 +133,21 @@ public void testSwitchToStrictMode() throws Exception {
assertNodeInCluster(remoteNodeName);
assertNodeInCluster(nonRemoteNodeName);

logger.info(" --> attempt switching to strict mode");
logger.info("Attempt switching to strict mode");
SettingsException exception = assertThrows(SettingsException.class, () -> setClusterMode(STRICT.mode));
assertEquals(
"can not switch to STRICT compatibility mode when the cluster contains both remote and non-remote nodes",
exception.getMessage()
);

logger.info(" --> stop remote node so that cluster had only non-remote nodes");
logger.info("Stop remote node so that cluster had only non-remote nodes");
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(remoteNodeName));
ensureStableCluster(2);

logger.info(" --> attempt switching to strict mode");
logger.info("Attempt switching to strict mode");
setClusterMode(STRICT.mode);
}

// restore indices from a snapshot
private void restoreSnapshot(String snapshotRepoName, String snapshotName, String restoredIndexName) {
RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepoName, snapshotName)
.setWaitForCompletion(false)
.setIndices(TEST_INDEX)
.setRenamePattern(TEST_INDEX)
.setRenameReplacement(restoredIndexName)
.get();

assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED);
ensureGreen(restoredIndexName);
}

// verify that the created index is not remote store backed
private void assertNonRemoteStoreBackedIndex(String indexName) {
Settings indexSettings = client.admin().indices().prepareGetIndex().execute().actionGet().getSettings().get(indexName);
assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
assertNull(indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
}

// verify that the created index is remote store backed
private void assertRemoteStoreBackedIndex(String indexName) {
Settings indexSettings = client.admin().indices().prepareGetIndex().execute().actionGet().getSettings().get(indexName);
assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
assertEquals("true", indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
assertEquals(REPOSITORY_NAME, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
assertEquals(REPOSITORY_2_NAME, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
assertEquals(
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)
);
}

// bootstrap a cluster
private void initializeCluster(boolean remoteClusterManager) {
addRemote = remoteClusterManager;
Expand Down
96 changes: 96 additions & 0 deletions .../java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,27 @@
package org.opensearch.remotemigration;

import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.cluster.node.DiscoveryNodes;
import org.opensearch.cluster.routing.IndexShardRoutingTable;
import org.opensearch.cluster.routing.ShardRouting;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.index.IndexSettings;
import org.opensearch.indices.replication.common.ReplicationType;
import org.opensearch.snapshots.SnapshotInfo;
import org.opensearch.snapshots.SnapshotState;

import java.util.Map;
import java.util.Optional;

import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
Expand Down Expand Up @@ -98,4 +110,88 @@ protected ShardRouting getShardRouting(boolean isPrimary) {
return (isPrimary ? table.primaryShard() : table.replicaShards().get(0));
}

// create a snapshot
public static SnapshotInfo createSnapshot(String snapshotRepoName, String snapshotName, String... indices) {
SnapshotInfo snapshotInfo = internalCluster().client()
.admin()
.cluster()
.prepareCreateSnapshot(snapshotRepoName, snapshotName)
.setIndices(indices)
.setWaitForCompletion(true)
.get()
.getSnapshotInfo();

assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
assertTrue(snapshotInfo.successfulShards() > 0);
assertEquals(0, snapshotInfo.failedShards());
return snapshotInfo;
}

// create new index
public static void createIndex(String indexName, int replicaCount) {
assertAcked(
internalCluster().client()
.admin()
.indices()
.prepareCreate(indexName)
.setSettings(
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicaCount)
.build()
)
.get()
);
}

// restore indices from a snapshot
public static RestoreSnapshotResponse restoreSnapshot(String snapshotRepoName, String snapshotName, String restoredIndexName) {
RestoreSnapshotResponse restoreSnapshotResponse = internalCluster().client()
.admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepoName, snapshotName)
.setWaitForCompletion(false)
.setIndices(TEST_INDEX)
.setRenamePattern(TEST_INDEX)
.setRenameReplacement(restoredIndexName)
.get();
assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED);
return restoreSnapshotResponse;
}

// verify that the created index is not remote store backed
public static void assertNonRemoteStoreBackedIndex(String indexName) {
Settings indexSettings = internalCluster().client()
.admin()
.indices()
.prepareGetIndex()
.execute()
.actionGet()
.getSettings()
.get(indexName);
assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
assertNull(indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
}

// verify that the created index is remote store backed
public static void assertRemoteStoreBackedIndex(String indexName) {
Settings indexSettings = internalCluster().client()
.admin()
.indices()
.prepareGetIndex()
.execute()
.actionGet()
.getSettings()
.get(indexName);
assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
assertEquals("true", indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
assertEquals(REPOSITORY_NAME, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
assertEquals(REPOSITORY_2_NAME, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
assertEquals(
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)
);
}
}
51 changes: 51 additions & 0 deletions ...internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,11 @@
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
import org.opensearch.client.Client;
import org.opensearch.common.settings.Settings;
import org.opensearch.repositories.blobstore.BlobStoreRepository;
import org.opensearch.snapshots.SnapshotInfo;
import org.opensearch.test.OpenSearchIntegTestCase;

import java.nio.file.Path;
import java.util.List;

import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
Expand Down Expand Up @@ -70,4 +73,52 @@ public void testMigrationDirections() {
updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), "random"));
assertThrows(IllegalArgumentException.class, () -> client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
}

public void testNoShallowSnapshotInMixedMode() throws Exception {
logger.info("Initialize remote cluster");
addRemote = true;
internalCluster().setBootstrapClusterManagerNodeIndex(0);
List<String> cmNodes = internalCluster().startNodes(1);
Client client = internalCluster().client(cmNodes.get(0));

logger.info("Add remote node");
internalCluster().startNode();
internalCluster().validateClusterFormed();

logger.info("Create remote backed index");
RemoteStoreMigrationShardAllocationBaseTestCase.createIndex("test", 0);
RemoteStoreMigrationShardAllocationBaseTestCase.assertRemoteStoreBackedIndex("test");

logger.info("Create shallow snapshot setting enabled repo");
String shallowSnapshotRepoName = "shallow-snapshot-repo-name";
Path shallowSnapshotRepoPath = randomRepoPath();
assertAcked(
clusterAdmin().preparePutRepository(shallowSnapshotRepoName)
.setType("fs")
.setSettings(
Settings.builder()
.put("location", shallowSnapshotRepoPath)
.put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), Boolean.TRUE)
)
);

logger.info("Verify shallow snapshot creation");
final String snapshot1 = "snapshot1";
SnapshotInfo snapshotInfo1 = RemoteStoreMigrationShardAllocationBaseTestCase.createSnapshot(
shallowSnapshotRepoName,
snapshot1,
"test"
);
assertEquals(snapshotInfo1.isRemoteStoreIndexShallowCopyEnabled(), true);

logger.info("Set MIXED compatibility mode");
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed"));
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());

logger.info("Verify that new snapshot is not shallow");
final String snapshot2 = "snapshot2";
SnapshotInfo snapshotInfo2 = RemoteStoreMigrationShardAllocationBaseTestCase.createSnapshot(shallowSnapshotRepoName, snapshot2);
assertEquals(snapshotInfo2.isRemoteStoreIndexShallowCopyEnabled(), false);
}
}
9 changes: 9 additions & 0 deletions server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,8 @@
import static java.util.Collections.emptySet;
import static java.util.Collections.unmodifiableList;
import static org.opensearch.cluster.SnapshotsInProgress.completed;
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode;
import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
import static org.opensearch.repositories.blobstore.BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY;
import static org.opensearch.snapshots.SnapshotUtils.validateSnapshotsBackingAnyIndex;

Expand Down Expand Up @@ -343,6 +345,13 @@ public ClusterState execute(ClusterState currentState) {
}

boolean remoteStoreIndexShallowCopy = REMOTE_STORE_INDEX_SHALLOW_COPY.get(repository.getMetadata().settings());
logger.debug("remote_store_index_shallow_copy setting is set as [{}]", remoteStoreIndexShallowCopy);
if (remoteStoreIndexShallowCopy
&& clusterService.getClusterSettings().get(REMOTE_STORE_COMPATIBILITY_MODE_SETTING).equals(CompatibilityMode.MIXED)) {
// don't allow shallow snapshots if compatibility mode is not strict
logger.warn("Shallow snapshots are not supported during migration. Falling back to full snapshot.");
remoteStoreIndexShallowCopy = false;
}
newEntry = SnapshotsInProgress.startedEntry(
new Snapshot(repositoryName, snapshotId),
request.includeGlobalState(),
Expand Down

0 comments on commit 5bf522c

Please sign in to comment.