From 0dea8c1e83b9450e07b4587dfbf591c69d0be098 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 27 Mar 2024 12:27:36 +0530 Subject: [PATCH] Add support to provide separate segment metadata repository Signed-off-by: Sachin Kale --- ...emoteStoreMockRepositoryIntegTestCase.java | 8 +- .../remotestore/CreateRemoteIndexIT.java | 28 ++--- .../remotestore/RemoteRestoreSnapshotIT.java | 8 +- .../RemoteStoreBaseIntegTestCase.java | 8 +- .../cluster/metadata/IndexMetadata.java | 113 ++++++++---------- .../metadata/MetadataCreateIndexService.java | 30 ++++- .../common/settings/IndexScopedSettings.java | 4 +- .../org/opensearch/index/IndexService.java | 3 +- .../org/opensearch/index/IndexSettings.java | 22 ++-- .../opensearch/index/shard/StoreRecovery.java | 1 + .../store/RemoteSegmentStoreDirectory.java | 1 + .../RemoteSegmentStoreDirectoryFactory.java | 48 +++++--- .../opensearch/indices/IndicesService.java | 4 +- .../remotestore/RemoteStoreNodeAttribute.java | 40 +++++-- .../blobstore/BlobStoreRepository.java | 2 +- .../TransportRemoteStoreStatsActionTests.java | 4 +- .../coordination/JoinTaskExecutorTests.java | 16 +-- .../MetadataCreateIndexServiceTests.java | 26 ++-- .../cluster/node/DiscoveryNodeTests.java | 2 +- ...eStoreMigrationAllocationDeciderTests.java | 8 +- .../opensearch/index/IndexModuleTests.java | 2 +- .../opensearch/index/IndexSettingsTests.java | 34 +++--- .../index/shard/IndexShardTests.java | 24 ++-- .../index/shard/RemoteIndexShardTests.java | 4 +- .../RemoteStoreRefreshListenerTests.java | 8 +- ...overyWithRemoteTranslogOnPrimaryTests.java | 2 +- ...moteSegmentStoreDirectoryFactoryTests.java | 4 +- .../RemoteSegmentStoreDirectoryTests.java | 4 +- ...teStorePeerRecoverySourceHandlerTests.java | 2 +- .../RemoteStoreReplicationSourceTests.java | 4 +- .../node/RemoteStoreNodeAttributeTests.java | 16 +-- .../BlobStoreRepositoryRemoteIndexTests.java | 8 +- .../index/shard/IndexShardTestCase.java | 4 +- .../test/OpenSearchIntegTestCase.java | 8 +- 34 files changed, 274 insertions(+), 226 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index 8166c0008ed83..fbd25fbf69249 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -31,8 +31,8 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; public abstract class AbstractRemoteStoreMockRepositoryIntegTestCase extends AbstractSnapshotIntegTestCase { @@ -89,7 +89,7 @@ public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailu ); return Settings.builder() - .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) + .put("node.attr." + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) .put(segmentRepoTypeAttributeKey, "mock") .put(segmentRepoSettingsAttributeKeyPrefix + "location", repoLocation) .put(segmentRepoSettingsAttributeKeyPrefix + "random_control_io_exception_rate", ioFailureRate) @@ -97,7 +97,7 @@ public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailu .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_list_blobs", true) .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_blobs", skipExceptionBlobList) .put(segmentRepoSettingsAttributeKeyPrefix + "max_failure_number", maxFailure) - .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, TRANSLOG_REPOSITORY_NAME) + .put("node.attr." + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, TRANSLOG_REPOSITORY_NAME) .put(translogRepoTypeAttributeKey, "mock") .put(translogRepoSettingsAttributeKeyPrefix + "location", repoLocation) .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java index d427a4db84ba2..984438c4df604 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java @@ -20,9 +20,9 @@ import java.util.Locale; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -149,7 +149,7 @@ public void testRemoteStoreSegmentRepoWithoutRemoteEnabledAndSegmentReplicationI Settings settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo") + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo") .build(); IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, @@ -161,7 +161,7 @@ public void testRemoteStoreSegmentRepoWithoutRemoteEnabledAndSegmentReplicationI String.format( Locale.ROOT, "Settings %s can only be set/enabled when %s is set to true", - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, SETTING_REMOTE_STORE_ENABLED ) ) @@ -174,7 +174,7 @@ public void testRemoteStoreEnabledByUserWithRemoteRepoIllegalArgumentException() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo") + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo") .build(); IllegalArgumentException exc = expectThrows( @@ -188,7 +188,7 @@ public void testRemoteStoreEnabledByUserWithRemoteRepoIllegalArgumentException() Locale.ROOT, "Validation Failed: 1: private index setting [%s] can not be set explicitly;2: private index setting [%s] can not be set explicitly;", SETTING_REMOTE_STORE_ENABLED, - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY + SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY ) ) ); @@ -198,7 +198,7 @@ public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() th Settings settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo") + .put(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "my-custom-repo") .build(); IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, @@ -210,7 +210,7 @@ public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() th String.format( Locale.ROOT, "Settings %s can only be set/enabled when %s is set to true", - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, + SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, SETTING_REMOTE_STORE_ENABLED ) ) @@ -223,8 +223,8 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo") - .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo") + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo") + .put(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "my-custom-repo") .build(); IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, @@ -237,8 +237,8 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception { Locale.ROOT, "Validation Failed: 1: private index setting [%s] can not be set explicitly;2: private index setting [%s] can not be set explicitly;3: private index setting [%s] can not be set explicitly;", SETTING_REMOTE_STORE_ENABLED, - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY + SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, + SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY ) ) ); @@ -254,8 +254,8 @@ protected void verifyRemoteStoreIndexSettings( ) { assertEquals(replicationType, indexSettings.get(SETTING_REPLICATION_TYPE)); assertEquals(isRemoteSegmentEnabled, indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); - assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY)); - assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY)); + assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY)); + assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY)); assertEquals(translogBufferInterval, INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index fff99e65054dc..ca96f0db5aafb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -51,7 +51,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -224,7 +224,7 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut .get(); indexSettings = getIndexResponse.settings().get(restoredIndexName1Seg); assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); - assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null)); + assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, null)); assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE)); assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1); // indexing some new docs and validating @@ -251,7 +251,7 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut .get(); indexSettings = getIndexResponse.settings().get(restoredIndexName1Doc); assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); - assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null)); + assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, null)); assertNull(indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE)); assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1); // indexing some new docs and validating @@ -571,7 +571,7 @@ public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS)); Settings remoteStoreIndexSettings = Settings.builder() - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo2Name) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, remoteStoreRepo2Name) .build(); // restore index as a remote store index with different remote store repo RestoreSnapshotResponse restoreSnapshotResponse = client.admin() diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index ba90cbe96e157..e78b787d9ed3d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -60,8 +60,8 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.repositories.fs.ReloadableFsRepository.REPOSITORIES_FAILRATE_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -288,10 +288,10 @@ public static Settings buildRemoteStoreNodeAttributes( ); Settings.Builder settings = Settings.builder() - .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) + .put("node.attr." + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) .put(segmentRepoTypeAttributeKey, segmentRepoType) .put(segmentRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) - .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) + .put("node.attr." + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) .put(translogRepoTypeAttributeKey, translogRepoType) .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath) .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 80b78cfe154f1..befe54c60fc74 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -321,9 +321,11 @@ public Iterator> settings() { public static final String SETTING_REMOTE_STORE_ENABLED = "index.remote_store.enabled"; - public static final String SETTING_REMOTE_SEGMENT_STORE_REPOSITORY = "index.remote_store.segment.repository"; + public static final String SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY = "index.remote_store.segment.repository"; - public static final String SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY = "index.remote_store.translog.repository"; + public static final String SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY = "index.remote_store.segment.metadata_repository"; + + public static final String SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY = "index.remote_store.translog.repository"; /** * Used to specify if the index data should be persisted in the remote store. @@ -362,46 +364,63 @@ public Iterator> settings() { Property.Dynamic ); - /** - * Used to specify remote store repository to use for this index. - */ - public static final Setting INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING = Setting.simpleString( - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, - new Setting.Validator<>() { + static class RepositoryNameValidator implements Setting.Validator { - @Override - public void validate(final String value) {} + String remoteStoreRepositorySettingKey; - @Override - public void validate(final String value, final Map, Object> settings) { - if (value == null || value.isEmpty()) { - throw new IllegalArgumentException( - "Setting " - + INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey() - + " should be provided with non-empty repository ID" - ); - } else { - validateRemoteStoreSettingEnabled(settings, INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING); - } - } + RepositoryNameValidator(String remoteStoreRepositorySettingKey) { + this.remoteStoreRepositorySettingKey = remoteStoreRepositorySettingKey; + } - @Override - public Iterator> settings() { - final List> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING); - return settings.iterator(); + @Override + public void validate(final String value) {} + + @Override + public void validate(final String value, final Map, Object> settings) { + if (value == null || value.isEmpty()) { + throw new IllegalArgumentException( + "Setting " + remoteStoreRepositorySettingKey + " should be provided with non-empty repository ID" + ); + } else { + validateRemoteStoreSettingEnabled(settings, remoteStoreRepositorySettingKey); } - }, + } + + @Override + public Iterator> settings() { + final List> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING); + return settings.iterator(); + } + } + + /** + * Used to specify remote store data repository to use for this index. + */ + public static final Setting INDEX_REMOTE_SEGMENT_STORE_DATA_REPOSITORY_SETTING = Setting.simpleString( + SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, + new RepositoryNameValidator(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY), + Property.IndexScope, + Property.PrivateIndex, + Property.Dynamic + ); + + /** + * Used to specify remote store metadata repository to use for this index. + */ + public static final Setting INDEX_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY_SETTING = Setting.simpleString( + SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY, + new RepositoryNameValidator(SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY), Property.IndexScope, Property.PrivateIndex, Property.Dynamic ); - private static void validateRemoteStoreSettingEnabled(final Map, Object> settings, Setting setting) { + private static void validateRemoteStoreSettingEnabled(final Map, Object> settings, String settingKey) { final Boolean isRemoteSegmentStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING); if (isRemoteSegmentStoreEnabled == false) { throw new IllegalArgumentException( "Settings " - + setting.getKey() + + settingKey + " can only be set/enabled when " + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey() + " is set to true" @@ -409,39 +428,9 @@ private static void validateRemoteStoreSettingEnabled(final Map, Obje } } - public static final Setting INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING = Setting.simpleString( - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, - new Setting.Validator<>() { - - @Override - public void validate(final String value) {} - - @Override - public void validate(final String value, final Map, Object> settings) { - if (value == null || value.isEmpty()) { - throw new IllegalArgumentException( - "Setting " + INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey() + " should be provided with non-empty repository ID" - ); - } else { - final Boolean isRemoteTranslogStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING); - if (isRemoteTranslogStoreEnabled == null || isRemoteTranslogStoreEnabled == false) { - throw new IllegalArgumentException( - "Settings " - + INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey() - + " can only be set/enabled when " - + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey() - + " is set to true" - ); - } - } - } - - @Override - public Iterator> settings() { - final List> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING); - return settings.iterator(); - } - }, + public static final Setting INDEX_REMOTE_TRANSLOG_DATA_REPOSITORY_SETTING = Setting.simpleString( + SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, + new RepositoryNameValidator(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY), Property.IndexScope, Property.PrivateIndex, Property.Dynamic diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index aee0473be95eb..487c0a959dd5e 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -134,9 +134,10 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING; import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; @@ -1023,17 +1024,34 @@ private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, if (isRemoteDataAttributePresent(clusterSettings)) { settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) .put( - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, clusterSettings.get( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY ) ) .put( - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, + SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, clusterSettings.get( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY ) ); + if (clusterSettings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY + ) != null) { + settingsBuilder.put( + SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY, + clusterSettings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY + ) + ); + } else { + settingsBuilder.put( + SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY, + clusterSettings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY + ) + ); + } } } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index c6c312d6b6eea..d5d43939a9147 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -228,8 +228,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // Settings for remote store enablement IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING, - IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING, - IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING, + IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_DATA_REPOSITORY_SETTING, + IndexMetadata.INDEX_REMOTE_TRANSLOG_DATA_REPOSITORY_SETTING, IndexSettings.INDEX_DOC_ID_FUZZY_SET_ENABLED_SETTING, IndexSettings.INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING, diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 109bda65b1fd8..2f380e0d318ae 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -505,7 +505,8 @@ public synchronized IndexShard createShard( seedRemote = true; } remoteDirectory = ((RemoteSegmentStoreDirectoryFactory) remoteDirectoryFactory).newDirectory( - RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(this.indexSettings.getNodeSettings()), + RemoteStoreNodeAttribute.getRemoteStoreSegmentDataRepo(this.indexSettings.getNodeSettings()), + RemoteStoreNodeAttribute.getRemoteStoreSegmentMetadataRepo(this.indexSettings.getNodeSettings()), this.indexSettings.getUUID(), shardId, this.indexSettings.getRemoteStorePathType() diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 7e3d812974c79..e4bcf57dc520d 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -726,8 +726,9 @@ public static IndexMergePolicy fromString(String text) { private final ReplicationType replicationType; private final boolean isRemoteStoreEnabled; private volatile TimeValue remoteTranslogUploadBufferInterval; - private final String remoteStoreTranslogRepository; - private final String remoteStoreRepository; + private final String remoteStoreTranslogDataRepository; + private final String remoteSegmentStoreDataRepository; + private final String remoteSegmentStoreMetadataRepository; private final boolean isRemoteSnapshot; private int remoteTranslogKeepExtraGen; private Version extendedCompatibilitySnapshotVersion; @@ -912,9 +913,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); replicationType = IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(settings); isRemoteStoreEnabled = settings.getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false); - remoteStoreTranslogRepository = settings.get(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY); + remoteStoreTranslogDataRepository = settings.get(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY); remoteTranslogUploadBufferInterval = INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings); - remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY); + remoteSegmentStoreDataRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY); + remoteSegmentStoreMetadataRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY); this.remoteTranslogKeepExtraGen = INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.get(settings); isRemoteSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match(this.settings); @@ -1263,12 +1265,16 @@ public boolean isRemoteTranslogStoreEnabled() { /** * Returns if remote store is enabled for this index. */ - public String getRemoteStoreRepository() { - return remoteStoreRepository; + public String getRemoteSegmentStoreDataRepository() { + return remoteSegmentStoreDataRepository; } - public String getRemoteStoreTranslogRepository() { - return remoteStoreTranslogRepository; + public String getRemoteSegmentStoreMetadataRepository() { + return remoteSegmentStoreMetadataRepository; + } + + public String getRemoteStoreTranslogDataRepository() { + return remoteStoreTranslogDataRepository; } /** diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index 9779a2320d79f..195d9c199692a 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -408,6 +408,7 @@ void recoverFromSnapshotAndRemoteStore( threadPool ); RemoteSegmentStoreDirectory sourceRemoteDirectory = (RemoteSegmentStoreDirectory) directoryFactory.newDirectory( + remoteStoreRepository, remoteStoreRepository, indexUUID, shardId, diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 999625e0e579f..b46b0b65a0150 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -903,6 +903,7 @@ public static void remoteDirectoryCleanup( ) { try { RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = (RemoteSegmentStoreDirectory) remoteDirectoryFactory.newDirectory( + remoteStoreRepoForIndex, remoteStoreRepoForIndex, indexUUID, shardId, diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index f0ecd96bcf1f7..a650615689c9a 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -50,37 +50,53 @@ public RemoteSegmentStoreDirectoryFactory(Supplier reposito @Override public Directory newDirectory(IndexSettings indexSettings, ShardPath path) throws IOException { - String repositoryName = indexSettings.getRemoteStoreRepository(); + String dataRepositoryName = indexSettings.getRemoteSegmentStoreDataRepository(); + String metadataRepositoryName = indexSettings.getRemoteSegmentStoreMetadataRepository(); String indexUUID = indexSettings.getIndex().getUUID(); - return newDirectory(repositoryName, indexUUID, path.getShardId(), indexSettings.getRemoteStorePathType()); + return newDirectory( + dataRepositoryName, + metadataRepositoryName, + indexUUID, + path.getShardId(), + indexSettings.getRemoteStorePathType() + ); } - public Directory newDirectory(String repositoryName, String indexUUID, ShardId shardId, RemoteStorePathType pathType) - throws IOException { + public Directory newDirectory( + String dataRepositoryName, + String metadataRepositoryName, + String indexUUID, + ShardId shardId, + RemoteStorePathType pathType + ) throws IOException { assert Objects.nonNull(pathType); - try (Repository repository = repositoriesService.get().repository(repositoryName)) { + try ( + Repository dataRepository = repositoriesService.get().repository(dataRepositoryName); + Repository metadataRepository = repositoriesService.get().repository(metadataRepositoryName) + ) { - assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; - BlobStoreRepository blobStoreRepository = ((BlobStoreRepository) repository); - BlobPath repositoryBasePath = blobStoreRepository.basePath(); + assert dataRepository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; + BlobStoreRepository blobStoreDataRepository = ((BlobStoreRepository) dataRepository); + BlobPath dataRepositoryBasePath = blobStoreDataRepository.basePath(); String shardIdStr = String.valueOf(shardId.id()); // Derive the path for data directory of SEGMENTS - BlobPath dataPath = pathType.path(repositoryBasePath, indexUUID, shardIdStr, SEGMENTS, DATA); + BlobPath dataPath = pathType.path(dataRepositoryBasePath, indexUUID, shardIdStr, SEGMENTS, DATA); RemoteDirectory dataDirectory = new RemoteDirectory( - blobStoreRepository.blobStore().blobContainer(dataPath), - blobStoreRepository::maybeRateLimitRemoteUploadTransfers, - blobStoreRepository::maybeRateLimitRemoteDownloadTransfers + blobStoreDataRepository.blobStore().blobContainer(dataPath), + blobStoreDataRepository::maybeRateLimitRemoteUploadTransfers, + blobStoreDataRepository::maybeRateLimitRemoteDownloadTransfers ); - // Derive the path for metadata directory of SEGMENTS - BlobPath mdPath = pathType.path(repositoryBasePath, indexUUID, shardIdStr, SEGMENTS, METADATA); - RemoteDirectory metadataDirectory = new RemoteDirectory(blobStoreRepository.blobStore().blobContainer(mdPath)); + BlobStoreRepository blobStoreMetadataRepository = ((BlobStoreRepository) metadataRepository); + BlobPath metadataRepositoryBasePath = blobStoreMetadataRepository.basePath(); + BlobPath mdPath = pathType.path(metadataRepositoryBasePath, indexUUID, shardIdStr, SEGMENTS, METADATA); + RemoteDirectory metadataDirectory = new RemoteDirectory(blobStoreDataRepository.blobStore().blobContainer(mdPath)); // The path for lock is derived within the RemoteStoreLockManagerFactory RemoteStoreLockManager mdLockManager = RemoteStoreLockManagerFactory.newLockManager( repositoriesService.get(), - repositoryName, + dataRepositoryName, indexUUID, shardIdStr, pathType diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 9bc81c1826c2d..6aba8da9a7169 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -548,14 +548,14 @@ private static BiFunction getTrans return new RemoteBlobStoreInternalTranslogFactory( repositoriesServiceSupplier, threadPool, - indexSettings.getRemoteStoreTranslogRepository(), + indexSettings.getRemoteStoreTranslogDataRepository(), remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardRouting.shardId()) ); } else if (isRemoteDataAttributePresent(settings) && shardRouting.primary()) { return new RemoteBlobStoreInternalTranslogFactory( repositoriesServiceSupplier, threadPool, - RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(indexSettings.getNodeSettings()), + RemoteStoreNodeAttribute.getRemoteStoreTranslogDataRepo(indexSettings.getNodeSettings()), remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardRouting.shardId()) ); } diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index a3bfe1195d8cc..0d5ccea2b5491 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -35,8 +35,9 @@ public class RemoteStoreNodeAttribute { public static final String REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX = "remote_store"; - public static final String REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.repository"; - public static final String REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.translog.repository"; + public static final String REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.repository"; + public static final String REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.metadata_repository"; + public static final String REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.translog.repository"; public static final String REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.state.repository"; public static final String REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s.type"; public static final String REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s." @@ -143,10 +144,13 @@ private RepositoriesMetadata buildRepositoriesMetadata(DiscoveryNode node) { private Set getValidatedRepositoryNames(DiscoveryNode node) { Set repositoryNames = new HashSet<>(); - if (node.getAttributes().containsKey(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY) - || node.getAttributes().containsKey(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY)) { - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY)); - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY)); + if (node.getAttributes().containsKey(REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY) + || node.getAttributes().containsKey(REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY)) { + repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY)); + if (node.getAttributes().get(REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY) != null) { + repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY)); + } + repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY)); repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)); } else if (node.getAttributes().containsKey(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)) { repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)); @@ -159,8 +163,10 @@ public static boolean isRemoteStoreAttributePresent(Settings settings) { } public static boolean isRemoteDataAttributePresent(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY).isEmpty() == false - || settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY).isEmpty() == false; + return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY) + .isEmpty() == false + || settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY) + .isEmpty() == false; } public static boolean isRemoteClusterStateAttributePresent(Settings settings) { @@ -168,12 +174,22 @@ public static boolean isRemoteClusterStateAttributePresent(Settings settings) { .isEmpty() == false; } - public static String getRemoteStoreSegmentRepo(Settings settings) { - return settings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); + public static String getRemoteStoreSegmentDataRepo(Settings settings) { + return settings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY + ); } - public static String getRemoteStoreTranslogRepo(Settings settings) { - return settings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); + public static String getRemoteStoreSegmentMetadataRepo(Settings settings) { + return settings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY + ); + } + + public static String getRemoteStoreTranslogDataRepo(Settings settings) { + return settings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY + ); } public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index a7c2fb03285b0..1689069ae0471 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -2701,7 +2701,7 @@ public void snapshotRemoteStoreIndexShard( indexTotalNumberOfFiles, indexTotalFileSize, store.indexSettings().getUUID(), - store.indexSettings().getRemoteStoreRepository(), + store.indexSettings().getRemoteSegmentStoreDataRepository(), this.basePath().toString(), fileNames ), diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java index ed73c2ef6ace5..928de1b473b51 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java @@ -41,7 +41,7 @@ import org.mockito.Mockito; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.mockito.ArgumentMatchers.any; @@ -74,7 +74,7 @@ public void setUp() throws Exception { settings(Version.CURRENT).put(SETTING_INDEX_UUID, INDEX.getUUID()) .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-test-repo") + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-test-repo") .build() ) .numberOfShards(2) diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index 5eafe63e63fad..75894be8ea92d 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -72,8 +72,8 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; import static org.opensearch.test.VersionUtils.allVersions; @@ -499,8 +499,8 @@ public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentAttributesJoin .build(); for (Map.Entry nodeAttribute : existingNodeAttributes.entrySet()) { - if (nodeAttribute.getKey() != REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY - && nodeAttribute.getKey() != REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY + if (nodeAttribute.getKey() != REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY + && nodeAttribute.getKey() != REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY && nodeAttribute.getKey() != REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY) { remoteStoreNodeAttributes.put(nodeAttribute.getKey(), nodeAttribute.getValue() + "-new"); validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); @@ -523,10 +523,10 @@ public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentNameAttributes .build(); for (Map.Entry nodeAttribute : existingNodeAttributes.entrySet()) { - if (REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { + if (REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO + "new", TRANSLOG_REPO); validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); - } else if (REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { + } else if (REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO + "new"); validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); } else if (REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { @@ -955,11 +955,11 @@ private Map remoteStoreNodeAttributes(String segmentRepoName, St return new HashMap<>() { { - put(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName); + put(REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName); put(segmentRepositoryTypeAttributeKey, "s3"); put(segmentRepositorySettingsAttributeKeyPrefix + "bucket", "segment_bucket"); put(segmentRepositorySettingsAttributeKeyPrefix + "base_path", "/segment/path"); - put(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName); + put(REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName); putIfAbsent(translogRepositoryTypeAttributeKey, "s3"); putIfAbsent(translogRepositorySettingsAttributeKeyPrefix + "bucket", "translog_bucket"); putIfAbsent(translogRepositorySettingsAttributeKeyPrefix + "base_path", "/translog/path"); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index cf4de32890a2a..dd22acb52be5c 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -124,9 +124,9 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.aggregateIndexSettings; @@ -147,8 +147,8 @@ import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.indices.ShardLimitValidatorTests.createTestShardLimitService; import static org.opensearch.node.Node.NODE_ATTRIBUTES; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; @@ -168,9 +168,9 @@ public class MetadataCreateIndexServiceTests extends OpenSearchTestCase { private QueryShardContext queryShardContext; private ClusterSettings clusterSettings; private static final String segmentRepositoryNameAttributeKey = NODE_ATTRIBUTES.getKey() - + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; + + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; private static final String translogRepositoryNameAttributeKey = NODE_ATTRIBUTES.getKey() - + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; + + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; final String REPLICATION_MISMATCH_VALIDATION_ERROR = "Validation Failed: 1: index setting [index.replication.type] is not allowed to be set as [cluster.index.restrict.replication.type=true];"; @@ -1474,7 +1474,7 @@ public void testRemoteStoreDisabledByUserIndexSettings() { public void testRemoteStoreOverrideSegmentRepoIndexSettings() { final Settings.Builder requestSettings = Settings.builder(); requestSettings.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo"); + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo"); withTemporaryClusterService(((clusterService, threadPool) -> { MetadataCreateIndexService checkerService = new MetadataCreateIndexService( Settings.EMPTY, @@ -1504,7 +1504,7 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { String.format( Locale.ROOT, "private index setting [%s] can not be set explicitly", - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY + SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY ) ) ); @@ -1513,7 +1513,7 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { public void testRemoteStoreOverrideTranslogRepoIndexSettings() { final Settings.Builder requestSettings = Settings.builder(); - requestSettings.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo"); + requestSettings.put(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "my-custom-repo"); withTemporaryClusterService(((clusterService, threadPool) -> { MetadataCreateIndexService checkerService = new MetadataCreateIndexService( Settings.EMPTY, @@ -1543,7 +1543,7 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() { String.format( Locale.ROOT, "private index setting [%s] can not be set explicitly", - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY + SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY ) ) ); @@ -1610,7 +1610,7 @@ public void testRemoteCustomData() { private IndexMetadata testRemoteCustomData(boolean remoteStoreEnabled, RemoteStorePathType remoteStorePathType) { Settings.Builder settingsBuilder = Settings.builder(); if (remoteStoreEnabled) { - settingsBuilder.put(NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "test"); + settingsBuilder.put(NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, "test"); } settingsBuilder.put(IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), remoteStorePathType.toString()); Settings settings = settingsBuilder.build(); @@ -2112,8 +2112,8 @@ private void verifyRemoteStoreIndexSettings( ) { assertEquals(replicationType, indexSettings.get(SETTING_REPLICATION_TYPE)); assertEquals(isRemoteSegmentEnabled, indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); - assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY)); - assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY)); + assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY)); + assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY)); assertEquals(translogBufferInterval, INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)); } diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java index c8a6fc76ce820..5607e90d28158 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java @@ -87,7 +87,7 @@ public void testRolesAreSorted() { public void testRemoteStoreRedactionInToString() { final Set roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)); Map attributes = new HashMap<>(); - attributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "test-repo"); + attributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, "test-repo"); attributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "test-repo"); final DiscoveryNode node = new DiscoveryNode( "name", diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java index 43363407d9249..cb3a50e6b1fb1 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java @@ -64,9 +64,9 @@ import java.util.Locale; import java.util.Map; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; @@ -603,8 +603,8 @@ private IndexMetadata.Builder getIndexMetadataBuilder(boolean isRemoteStoreBacke Settings.Builder builder = settings(Version.CURRENT); if (isRemoteStoreBackedIndex) { builder.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, TEST_REPO) - .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, TEST_REPO) + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, TEST_REPO) + .put(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, TEST_REPO) .put(SETTING_REMOTE_STORE_ENABLED, true); } return IndexMetadata.builder(TEST_INDEX).settings(builder).numberOfShards(shardCount).numberOfReplicas(replicaCount); diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 97bc822be7d51..5be61eec13d6e 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -235,7 +235,7 @@ private IndexService newIndexService(IndexModule module) throws IOException { return new RemoteBlobStoreInternalTranslogFactory( repositoriesServiceReference::get, threadPool, - indexSettings.getRemoteStoreTranslogRepository(), + indexSettings.getRemoteStoreTranslogDataRepository(), new RemoteTranslogTransferTracker(shardRouting.shardId(), 10) ); } diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index e4ce879a5ec5e..00f0a70338f8e 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -823,7 +823,7 @@ public void testRemoteRepositoryDefaultSetting() { Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertNull(settings.getRemoteStoreRepository()); + assertNull(settings.getRemoteSegmentStoreDataRepository()); } public void testRemoteRepositoryExplicitSetting() { @@ -833,28 +833,28 @@ public void testRemoteRepositoryExplicitSetting() { .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "repo1") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "repo1") .build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertEquals("repo1", settings.getRemoteStoreRepository()); + assertEquals("repo1", settings.getRemoteSegmentStoreDataRepository()); } public void testSetRemoteRepositoryFailsWhenRemoteStoreIsNotEnabled() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "repo1") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "repo1") .build(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, - () -> IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(indexSettings) + () -> IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_DATA_REPOSITORY_SETTING.get(indexSettings) ); assertEquals( String.format( Locale.ROOT, "Settings %s can only be set/enabled when %s is set to true", - IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, IndexMetadata.SETTING_REMOTE_STORE_ENABLED ), iae.getMessage() @@ -865,17 +865,17 @@ public void testSetRemoteRepositoryFailsWhenEmptyString() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "") .build(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, - () -> IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(indexSettings) + () -> IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_DATA_REPOSITORY_SETTING.get(indexSettings) ); assertEquals( String.format( Locale.ROOT, "Setting %s should be provided with non-empty repository ID", - IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY + IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY ), iae.getMessage() ); @@ -887,7 +887,7 @@ public void testRemoteTranslogRepoDefaultSetting() { Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertNull(settings.getRemoteStoreRepository()); + assertNull(settings.getRemoteSegmentStoreDataRepository()); } public void testRemoteTranslogExplicitSetting() { @@ -895,13 +895,13 @@ public void testRemoteTranslogExplicitSetting() { "index", Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "tlog-store") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "tlog-store") .put(IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "200ms") .build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertNull(settings.getRemoteStoreRepository()); - assertEquals("tlog-store", settings.getRemoteStoreTranslogRepository()); + assertNull(settings.getRemoteSegmentStoreDataRepository()); + assertEquals("tlog-store", settings.getRemoteStoreTranslogDataRepository()); assertEquals(TimeValue.timeValueMillis(200), settings.getRemoteTranslogUploadBufferInterval()); } @@ -909,11 +909,11 @@ public void testSetRemoteTranslogRepositoryFailsWhenRemoteStoreIsNotEnabled() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "repo1") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "repo1") .build(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, - () -> IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(indexSettings) + () -> IndexMetadata.INDEX_REMOTE_TRANSLOG_DATA_REPOSITORY_SETTING.get(indexSettings) ); assertEquals( "Settings index.remote_store.translog.repository can only be set/enabled when index.remote_store.enabled is set to true", @@ -925,11 +925,11 @@ public void testSetRemoteTranslogRepositoryFailsWhenEmptyString() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "") .build(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, - () -> IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(indexSettings) + () -> IndexMetadata.INDEX_REMOTE_TRANSLOG_DATA_REPOSITORY_SETTING.get(indexSettings) ); assertEquals("Setting index.remote_store.translog.repository should be provided with non-empty repository ID", iae.getMessage()); } diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index 537bfcf8f8a6b..388ea50918699 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -2730,8 +2730,8 @@ public void testRelocatedForRemoteTranslogBackedIndexWithAsyncDurability() throw .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "seg-test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "txlog-test") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "seg-test") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "txlog-test") .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC) .build(); final IndexShard indexShard = newStartedShard(true, settings, new NRTReplicationEngineFactory()); @@ -2842,8 +2842,8 @@ public void testSyncSegmentsFromGivenRemoteSegmentStore() throws IOException { Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStorePath + "__test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStorePath + "__test") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, remoteStorePath + "__test") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, remoteStorePath + "__test") .build(), new InternalEngineFactory() ); @@ -2883,8 +2883,8 @@ public void testSyncSegmentsFromGivenRemoteSegmentStore() throws IOException { Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStorePath + "__test1") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStorePath + "__test1") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, remoteStorePath + "__test1") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, remoteStorePath + "__test1") .build(), new InternalEngineFactory() ); @@ -2926,8 +2926,8 @@ public void testRestoreShardFromRemoteStore(boolean performFlush) throws IOExcep Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStorePath + "__test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStorePath + "__test") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, remoteStorePath + "__test") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, remoteStorePath + "__test") .build(), new InternalEngineFactory() ); @@ -4814,8 +4814,8 @@ public void testTranslogFactoryForRemoteTranslogBackedPrimaryShard() throws IOEx .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "seg-test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "txlog-test") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "seg-test") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "txlog-test") .build(); final IndexShard primaryShard = newStartedShard(true, primarySettings, new NRTReplicationEngineFactory()); assertEquals(primaryShard.getEngine().getClass(), InternalEngine.class); @@ -4830,8 +4830,8 @@ public void testTranslogFactoryForRemoteTranslogBackedReplicaShard() throws IOEx .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "seg-test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "txlog-test") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "seg-test") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "txlog-test") .build(); final IndexShard replicaShard = newStartedShard(false, primarySettings, new NRTReplicationEngineFactory()); assertEquals(replicaShard.getEngine().getClass(), NRTReplicationEngine.class); diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java index 57a561bc8f2a3..9acdc7bcb616d 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java @@ -64,8 +64,8 @@ public class RemoteIndexShardTests extends SegmentReplicationIndexShardTests { private static final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, REPOSITORY_NAME) .build(); protected Settings getIndexSettings() { diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 85878cc2e1c9d..864172cad01b4 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -70,8 +70,8 @@ public void setup(boolean primary, int numberOfDocs) throws IOException { primary, Settings.builder() .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "temp-fs") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "temp-fs") .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(), new InternalEngineFactory() @@ -537,8 +537,8 @@ private Tuple mockIn true, Settings.builder() .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "temp-fs") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "temp-fs") .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(), new InternalEngineFactory() diff --git a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java index 85864eebd6d0d..80ae0ccee4c0a 100644 --- a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java @@ -37,7 +37,7 @@ public class ReplicaRecoveryWithRemoteTranslogOnPrimaryTests extends OpenSearchI private static final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "translog-repo") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "translog-repo") .put(IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "100ms") .build(); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java index cad5e47531cc6..58e5eb1c45ba6 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java @@ -63,7 +63,7 @@ public void setup() { public void testNewDirectory() throws IOException { Settings settings = Settings.builder() .put(IndexMetadata.SETTING_INDEX_UUID, "uuid_1") - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "remote_store_repository") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "remote_store_repository") .build(); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings); Path tempDir = createTempDir().resolve(indexSettings.getUUID()).resolve("0"); @@ -109,7 +109,7 @@ public void testNewDirectory() throws IOException { public void testNewDirectoryRepositoryDoesNotExist() { Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "remote_store_repository") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "remote_store_repository") .build(); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings); Path tempDir = createTempDir().resolve(indexSettings.getUUID()).resolve("0"); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 59c8a9d92f07b..f572c0d5e8443 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -698,7 +698,7 @@ public void testCleanupAsync() throws Exception { threadPool, indexShard.shardId() ); - when(remoteSegmentStoreDirectoryFactory.newDirectory(any(), any(), any(), any())).thenReturn(remoteSegmentDirectory); + when(remoteSegmentStoreDirectoryFactory.newDirectory(any(), any(), any(), any(), any())).thenReturn(remoteSegmentDirectory); String repositoryName = "test-repository"; String indexUUID = "test-idx-uuid"; ShardId shardId = new ShardId(Index.UNKNOWN_INDEX_NAME, indexUUID, Integer.parseInt("0")); @@ -711,7 +711,7 @@ public void testCleanupAsync() throws Exception { shardId, pathType ); - verify(remoteSegmentStoreDirectoryFactory).newDirectory(repositoryName, indexUUID, shardId, pathType); + verify(remoteSegmentStoreDirectoryFactory).newDirectory(repositoryName, repositoryName, indexUUID, shardId, pathType); verify(threadPool, times(0)).executor(ThreadPool.Names.REMOTE_PURGE); verify(remoteMetadataDirectory).delete(); verify(remoteDataDirectory).delete(); diff --git a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java index 131514eb019b3..beaf6b071cfe1 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java @@ -25,7 +25,7 @@ public class RemoteStorePeerRecoverySourceHandlerTests extends OpenSearchIndexLe private static final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "translog-repo") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "translog-repo") .put(IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "100ms") .build(); diff --git a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java index 287962b158c79..c619fc6c401e2 100644 --- a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java @@ -42,8 +42,8 @@ public class RemoteStoreReplicationSourceTests extends OpenSearchIndexLevelRepli private IndexShard replicaShard; private final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-repo") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-repo") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-repo") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "my-repo") .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); diff --git a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java index c4ba271d27ae9..7bdb103f226d0 100644 --- a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java +++ b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java @@ -28,8 +28,8 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_CRYPTO_SETTINGS_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; public class RemoteStoreNodeAttributeTests extends OpenSearchTestCase { @@ -43,9 +43,9 @@ public void testCryptoMetadata() throws UnknownHostException { String repoCryptoMetadataKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, repoName); String repoCryptoMetadataSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_CRYPTO_SETTINGS_PREFIX, repoName); Map attr = Map.of( - REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, - REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, @@ -89,9 +89,9 @@ public void testInvalidCryptoMetadata() throws UnknownHostException { String repoCryptoMetadataKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, repoName); String repoCryptoMetadataSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_CRYPTO_SETTINGS_PREFIX, repoName); Map attr = Map.of( - REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, - REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, @@ -122,9 +122,9 @@ public void testNoCryptoMetadata() throws UnknownHostException { String repoTypeSettingKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, repoName); String repoSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, repoName); Map attr = Map.of( - REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, - REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index 9cca495cced72..5e1940d1769fc 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -62,8 +62,8 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.hamcrest.Matchers.equalTo; /** @@ -97,10 +97,10 @@ private Settings buildRemoteStoreNodeAttributes(String repoName, Path repoPath) ); return Settings.builder() - .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) + .put("node.attr." + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) .put(repoTypeAttributeKey, FsRepository.TYPE) .put(repoSettingsAttributeKeyPrefix + "location", repoPath) - .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) + .put("node.attr." + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) .put(repoTypeAttributeKey, FsRepository.TYPE) .put(repoSettingsAttributeKeyPrefix + "location", repoPath) .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index a2f9eb677c0ac..41a1b9213348b 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -654,7 +654,7 @@ protected IndexShard newShard( RepositoriesService mockRepoSvc = mock(RepositoriesService.class); if (indexSettings.isRemoteStoreEnabled() || indexSettings.isRemoteNode()) { - String remoteStoreRepository = indexSettings.getRemoteStoreRepository(); + String remoteStoreRepository = indexSettings.getRemoteSegmentStoreDataRepository(); // remote path via setting a repository . This is a hack used for shards are created using reset . // since we can't get remote path from IndexShard directly, we are using repository to store it . if (remoteStoreRepository != null && remoteStoreRepository.endsWith("__test")) { @@ -677,7 +677,7 @@ protected IndexShard newShard( return new RemoteBlobStoreInternalTranslogFactory( () -> mockRepoSvc, threadPool, - settings.getRemoteStoreTranslogRepository(), + settings.getRemoteStoreTranslogDataRepository(), new RemoteTranslogTransferTracker(shardRouting.shardId(), 20) ); } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 664314245530e..8fa8635320b6a 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -214,8 +214,8 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.test.XContentTestUtils.convertToMap; import static org.opensearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -2579,10 +2579,10 @@ public static Settings buildRemoteStoreNodeAttributes( ); Settings.Builder settings = Settings.builder() - .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) + .put("node.attr." + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) .put(segmentRepoTypeAttributeKey, segmentRepoType) .put(segmentRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) - .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) + .put("node.attr." + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) .put(translogRepoTypeAttributeKey, translogRepoType) .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath) .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName)