Skip to content

Commit

Permalink
Add support to provide separate segment metadata repository
Browse files Browse the repository at this point in the history
Signed-off-by: Sachin Kale <[email protected]>
  • Loading branch information
Sachin Kale committed Apr 1, 2024
1 parent 618782d commit 0dea8c1
Show file tree
Hide file tree
Showing 34 changed files with 274 additions and 226 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY;

public abstract class AbstractRemoteStoreMockRepositoryIntegTestCase extends AbstractSnapshotIntegTestCase {

Expand Down Expand Up @@ -89,15 +89,15 @@ public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailu
);

return Settings.builder()
.put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME)
.put("node.attr." + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME)
.put(segmentRepoTypeAttributeKey, "mock")
.put(segmentRepoSettingsAttributeKeyPrefix + "location", repoLocation)
.put(segmentRepoSettingsAttributeKeyPrefix + "random_control_io_exception_rate", ioFailureRate)
.put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_verification_file", true)
.put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_list_blobs", true)
.put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_blobs", skipExceptionBlobList)
.put(segmentRepoSettingsAttributeKeyPrefix + "max_failure_number", maxFailure)
.put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, TRANSLOG_REPOSITORY_NAME)
.put("node.attr." + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, TRANSLOG_REPOSITORY_NAME)
.put(translogRepoTypeAttributeKey, "mock")
.put(translogRepoSettingsAttributeKeyPrefix + "location", repoLocation)
.put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@

import java.util.Locale;

import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
Expand Down Expand Up @@ -149,7 +149,7 @@ public void testRemoteStoreSegmentRepoWithoutRemoteEnabledAndSegmentReplicationI
Settings settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo")
.put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo")
.build();
IllegalArgumentException exc = expectThrows(
IllegalArgumentException.class,
Expand All @@ -161,7 +161,7 @@ public void testRemoteStoreSegmentRepoWithoutRemoteEnabledAndSegmentReplicationI
String.format(
Locale.ROOT,
"Settings %s can only be set/enabled when %s is set to true",
SETTING_REMOTE_SEGMENT_STORE_REPOSITORY,
SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY,
SETTING_REMOTE_STORE_ENABLED
)
)
Expand All @@ -174,7 +174,7 @@ public void testRemoteStoreEnabledByUserWithRemoteRepoIllegalArgumentException()
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
.put(SETTING_REMOTE_STORE_ENABLED, true)
.put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo")
.put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo")
.build();

IllegalArgumentException exc = expectThrows(
Expand All @@ -188,7 +188,7 @@ public void testRemoteStoreEnabledByUserWithRemoteRepoIllegalArgumentException()
Locale.ROOT,
"Validation Failed: 1: private index setting [%s] can not be set explicitly;2: private index setting [%s] can not be set explicitly;",
SETTING_REMOTE_STORE_ENABLED,
SETTING_REMOTE_SEGMENT_STORE_REPOSITORY
SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY
)
)
);
Expand All @@ -198,7 +198,7 @@ public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() th
Settings settings = Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo")
.put(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "my-custom-repo")
.build();
IllegalArgumentException exc = expectThrows(
IllegalArgumentException.class,
Expand All @@ -210,7 +210,7 @@ public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() th
String.format(
Locale.ROOT,
"Settings %s can only be set/enabled when %s is set to true",
SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY,
SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY,
SETTING_REMOTE_STORE_ENABLED
)
)
Expand All @@ -223,8 +223,8 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception {
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
.put(SETTING_REMOTE_STORE_ENABLED, true)
.put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo")
.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo")
.put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo")
.put(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "my-custom-repo")
.build();
IllegalArgumentException exc = expectThrows(
IllegalArgumentException.class,
Expand All @@ -237,8 +237,8 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception {
Locale.ROOT,
"Validation Failed: 1: private index setting [%s] can not be set explicitly;2: private index setting [%s] can not be set explicitly;3: private index setting [%s] can not be set explicitly;",
SETTING_REMOTE_STORE_ENABLED,
SETTING_REMOTE_SEGMENT_STORE_REPOSITORY,
SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY
SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY,
SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY
)
)
);
Expand All @@ -254,8 +254,8 @@ protected void verifyRemoteStoreIndexSettings(
) {
assertEquals(replicationType, indexSettings.get(SETTING_REPLICATION_TYPE));
assertEquals(isRemoteSegmentEnabled, indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY));
assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY));
assertEquals(translogBufferInterval, INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
Expand Down Expand Up @@ -224,7 +224,7 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut
.get();
indexSettings = getIndexResponse.settings().get(restoredIndexName1Seg);
assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null));
assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, null));
assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE));
assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1);
// indexing some new docs and validating
Expand All @@ -251,7 +251,7 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut
.get();
indexSettings = getIndexResponse.settings().get(restoredIndexName1Doc);
assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null));
assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, null));
assertNull(indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE));
assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1);
// indexing some new docs and validating
Expand Down Expand Up @@ -571,7 +571,7 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException
assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS));

Settings remoteStoreIndexSettings = Settings.builder()
.put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo2Name)
.put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, remoteStoreRepo2Name)
.build();
// restore index as a remote store index with different remote store repo
RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY;
import static org.opensearch.repositories.fs.ReloadableFsRepository.REPOSITORIES_FAILRATE_SETTING;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;

Expand Down Expand Up @@ -288,10 +288,10 @@ public static Settings buildRemoteStoreNodeAttributes(
);

Settings.Builder settings = Settings.builder()
.put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName)
.put("node.attr." + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName)
.put(segmentRepoTypeAttributeKey, segmentRepoType)
.put(segmentRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath)
.put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName)
.put("node.attr." + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName)
.put(translogRepoTypeAttributeKey, translogRepoType)
.put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath)
.put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -321,9 +321,11 @@ public Iterator<Setting<?>> settings() {

public static final String SETTING_REMOTE_STORE_ENABLED = "index.remote_store.enabled";

public static final String SETTING_REMOTE_SEGMENT_STORE_REPOSITORY = "index.remote_store.segment.repository";
public static final String SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY = "index.remote_store.segment.repository";

public static final String SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY = "index.remote_store.translog.repository";
public static final String SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY = "index.remote_store.segment.metadata_repository";

public static final String SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY = "index.remote_store.translog.repository";

/**
* Used to specify if the index data should be persisted in the remote store.
Expand Down Expand Up @@ -362,86 +364,73 @@ public Iterator<Setting<?>> settings() {
Property.Dynamic
);

/**
* Used to specify remote store repository to use for this index.
*/
public static final Setting<String> INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING = Setting.simpleString(
SETTING_REMOTE_SEGMENT_STORE_REPOSITORY,
new Setting.Validator<>() {
static class RepositoryNameValidator implements Setting.Validator<String> {

@Override
public void validate(final String value) {}
String remoteStoreRepositorySettingKey;

@Override
public void validate(final String value, final Map<Setting<?>, Object> settings) {
if (value == null || value.isEmpty()) {
throw new IllegalArgumentException(
"Setting "
+ INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey()
+ " should be provided with non-empty repository ID"
);
} else {
validateRemoteStoreSettingEnabled(settings, INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING);
}
}
RepositoryNameValidator(String remoteStoreRepositorySettingKey) {
this.remoteStoreRepositorySettingKey = remoteStoreRepositorySettingKey;
}

@Override
public Iterator<Setting<?>> settings() {
final List<Setting<?>> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING);
return settings.iterator();
@Override
public void validate(final String value) {}

@Override
public void validate(final String value, final Map<Setting<?>, Object> settings) {
if (value == null || value.isEmpty()) {
throw new IllegalArgumentException(
"Setting " + remoteStoreRepositorySettingKey + " should be provided with non-empty repository ID"
);
} else {
validateRemoteStoreSettingEnabled(settings, remoteStoreRepositorySettingKey);
}
},
}

@Override
public Iterator<Setting<?>> settings() {
final List<Setting<?>> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING);
return settings.iterator();
}
}

/**
* Used to specify remote store data repository to use for this index.
*/
public static final Setting<String> INDEX_REMOTE_SEGMENT_STORE_DATA_REPOSITORY_SETTING = Setting.simpleString(
SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY,
new RepositoryNameValidator(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY),
Property.IndexScope,
Property.PrivateIndex,
Property.Dynamic
);

/**
* Used to specify remote store metadata repository to use for this index.
*/
public static final Setting<String> INDEX_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY_SETTING = Setting.simpleString(
SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY,
new RepositoryNameValidator(SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY),
Property.IndexScope,
Property.PrivateIndex,
Property.Dynamic
);

private static void validateRemoteStoreSettingEnabled(final Map<Setting<?>, Object> settings, Setting<?> setting) {
private static void validateRemoteStoreSettingEnabled(final Map<Setting<?>, Object> settings, String settingKey) {
final Boolean isRemoteSegmentStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING);
if (isRemoteSegmentStoreEnabled == false) {
throw new IllegalArgumentException(
"Settings "
+ setting.getKey()
+ settingKey
+ " can only be set/enabled when "
+ INDEX_REMOTE_STORE_ENABLED_SETTING.getKey()
+ " is set to true"
);
}
}

public static final Setting<String> INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING = Setting.simpleString(
SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY,
new Setting.Validator<>() {

@Override
public void validate(final String value) {}

@Override
public void validate(final String value, final Map<Setting<?>, Object> settings) {
if (value == null || value.isEmpty()) {
throw new IllegalArgumentException(
"Setting " + INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey() + " should be provided with non-empty repository ID"
);
} else {
final Boolean isRemoteTranslogStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING);
if (isRemoteTranslogStoreEnabled == null || isRemoteTranslogStoreEnabled == false) {
throw new IllegalArgumentException(
"Settings "
+ INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey()
+ " can only be set/enabled when "
+ INDEX_REMOTE_STORE_ENABLED_SETTING.getKey()
+ " is set to true"
);
}
}
}

@Override
public Iterator<Setting<?>> settings() {
final List<Setting<?>> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING);
return settings.iterator();
}
},
public static final Setting<String> INDEX_REMOTE_TRANSLOG_DATA_REPOSITORY_SETTING = Setting.simpleString(
SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY,
new RepositoryNameValidator(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY),
Property.IndexScope,
Property.PrivateIndex,
Property.Dynamic
Expand Down
Loading

0 comments on commit 0dea8c1

Please sign in to comment.