From f262f3908f4a3e03ba2e7292ede9ce06fcca6f5a Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 27 Mar 2024 12:27:36 +0530 Subject: [PATCH 1/8] Add support to provide separate segment metadata repository Signed-off-by: Sachin Kale --- ...emoteStoreMockRepositoryIntegTestCase.java | 8 +- .../remotestore/CreateRemoteIndexIT.java | 28 ++--- .../remotestore/RemoteRestoreSnapshotIT.java | 7 +- .../cluster/metadata/IndexMetadata.java | 113 ++++++++---------- .../metadata/MetadataCreateIndexService.java | 28 +++-- .../common/settings/IndexScopedSettings.java | 4 +- .../org/opensearch/index/IndexService.java | 3 +- .../org/opensearch/index/IndexSettings.java | 22 ++-- .../opensearch/index/shard/StoreRecovery.java | 1 + .../store/RemoteSegmentStoreDirectory.java | 1 + .../RemoteSegmentStoreDirectoryFactory.java | 48 +++++--- .../opensearch/indices/IndicesService.java | 4 +- .../remotestore/RemoteStoreNodeAttribute.java | 40 +++++-- .../blobstore/BlobStoreRepository.java | 2 +- .../TransportRemoteStoreStatsActionTests.java | 4 +- .../coordination/JoinTaskExecutorTests.java | 16 +-- .../MetadataCreateIndexServiceTests.java | 26 ++-- .../cluster/node/DiscoveryNodeTests.java | 2 +- ...eStoreMigrationAllocationDeciderTests.java | 8 +- .../opensearch/index/IndexModuleTests.java | 2 +- .../opensearch/index/IndexSettingsTests.java | 34 +++--- .../index/shard/IndexShardTests.java | 24 ++-- .../index/shard/RemoteIndexShardTests.java | 4 +- .../RemoteStoreRefreshListenerTests.java | 8 +- ...overyWithRemoteTranslogOnPrimaryTests.java | 2 +- ...moteSegmentStoreDirectoryFactoryTests.java | 4 +- .../RemoteSegmentStoreDirectoryTests.java | 5 +- ...teStorePeerRecoverySourceHandlerTests.java | 2 +- .../RemoteStoreReplicationSourceTests.java | 4 +- .../node/RemoteStoreNodeAttributeTests.java | 16 +-- .../BlobStoreRepositoryRemoteIndexTests.java | 8 +- .../index/shard/IndexShardTestCase.java | 4 +- .../test/OpenSearchIntegTestCase.java | 8 +- 33 files changed, 265 insertions(+), 225 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index 8166c0008ed83..fbd25fbf69249 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -31,8 +31,8 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; public abstract class AbstractRemoteStoreMockRepositoryIntegTestCase extends AbstractSnapshotIntegTestCase { @@ -89,7 +89,7 @@ public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailu ); return Settings.builder() - .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) + .put("node.attr." + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) .put(segmentRepoTypeAttributeKey, "mock") .put(segmentRepoSettingsAttributeKeyPrefix + "location", repoLocation) .put(segmentRepoSettingsAttributeKeyPrefix + "random_control_io_exception_rate", ioFailureRate) @@ -97,7 +97,7 @@ public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailu .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_list_blobs", true) .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_blobs", skipExceptionBlobList) .put(segmentRepoSettingsAttributeKeyPrefix + "max_failure_number", maxFailure) - .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, TRANSLOG_REPOSITORY_NAME) + .put("node.attr." + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, TRANSLOG_REPOSITORY_NAME) .put(translogRepoTypeAttributeKey, "mock") .put(translogRepoSettingsAttributeKeyPrefix + "location", repoLocation) .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java index d427a4db84ba2..984438c4df604 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java @@ -20,9 +20,9 @@ import java.util.Locale; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -149,7 +149,7 @@ public void testRemoteStoreSegmentRepoWithoutRemoteEnabledAndSegmentReplicationI Settings settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo") + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo") .build(); IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, @@ -161,7 +161,7 @@ public void testRemoteStoreSegmentRepoWithoutRemoteEnabledAndSegmentReplicationI String.format( Locale.ROOT, "Settings %s can only be set/enabled when %s is set to true", - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, SETTING_REMOTE_STORE_ENABLED ) ) @@ -174,7 +174,7 @@ public void testRemoteStoreEnabledByUserWithRemoteRepoIllegalArgumentException() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo") + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo") .build(); IllegalArgumentException exc = expectThrows( @@ -188,7 +188,7 @@ public void testRemoteStoreEnabledByUserWithRemoteRepoIllegalArgumentException() Locale.ROOT, "Validation Failed: 1: private index setting [%s] can not be set explicitly;2: private index setting [%s] can not be set explicitly;", SETTING_REMOTE_STORE_ENABLED, - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY + SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY ) ) ); @@ -198,7 +198,7 @@ public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() th Settings settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo") + .put(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "my-custom-repo") .build(); IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, @@ -210,7 +210,7 @@ public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() th String.format( Locale.ROOT, "Settings %s can only be set/enabled when %s is set to true", - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, + SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, SETTING_REMOTE_STORE_ENABLED ) ) @@ -223,8 +223,8 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception { .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo") - .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo") + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo") + .put(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "my-custom-repo") .build(); IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, @@ -237,8 +237,8 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception { Locale.ROOT, "Validation Failed: 1: private index setting [%s] can not be set explicitly;2: private index setting [%s] can not be set explicitly;3: private index setting [%s] can not be set explicitly;", SETTING_REMOTE_STORE_ENABLED, - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY + SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, + SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY ) ) ); @@ -254,8 +254,8 @@ protected void verifyRemoteStoreIndexSettings( ) { assertEquals(replicationType, indexSettings.get(SETTING_REPLICATION_TYPE)); assertEquals(isRemoteSegmentEnabled, indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); - assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY)); - assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY)); + assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY)); + assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY)); assertEquals(translogBufferInterval, INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index d34a5f4edbaec..297f3816ba395 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -54,6 +54,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; @@ -710,7 +711,7 @@ public void testInvalidRestoreRequestScenarios() throws Exception { // try index restore with remote store repository modified Settings remoteStoreIndexSettings = Settings.builder() - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, newRemoteStoreRepo) .build(); exception = expectThrows( @@ -735,8 +736,8 @@ public void testInvalidRestoreRequestScenarios() throws Exception { .prepareRestoreSnapshot(snapshotRepo, snapshotName1) .setWaitForCompletion(false) .setIgnoreIndexSettings( - IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, - IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY + IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, + IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY ) .setIndices(index) .setRenamePattern(index) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 80b78cfe154f1..befe54c60fc74 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -321,9 +321,11 @@ public Iterator> settings() { public static final String SETTING_REMOTE_STORE_ENABLED = "index.remote_store.enabled"; - public static final String SETTING_REMOTE_SEGMENT_STORE_REPOSITORY = "index.remote_store.segment.repository"; + public static final String SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY = "index.remote_store.segment.repository"; - public static final String SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY = "index.remote_store.translog.repository"; + public static final String SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY = "index.remote_store.segment.metadata_repository"; + + public static final String SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY = "index.remote_store.translog.repository"; /** * Used to specify if the index data should be persisted in the remote store. @@ -362,46 +364,63 @@ public Iterator> settings() { Property.Dynamic ); - /** - * Used to specify remote store repository to use for this index. - */ - public static final Setting INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING = Setting.simpleString( - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, - new Setting.Validator<>() { + static class RepositoryNameValidator implements Setting.Validator { - @Override - public void validate(final String value) {} + String remoteStoreRepositorySettingKey; - @Override - public void validate(final String value, final Map, Object> settings) { - if (value == null || value.isEmpty()) { - throw new IllegalArgumentException( - "Setting " - + INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey() - + " should be provided with non-empty repository ID" - ); - } else { - validateRemoteStoreSettingEnabled(settings, INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING); - } - } + RepositoryNameValidator(String remoteStoreRepositorySettingKey) { + this.remoteStoreRepositorySettingKey = remoteStoreRepositorySettingKey; + } - @Override - public Iterator> settings() { - final List> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING); - return settings.iterator(); + @Override + public void validate(final String value) {} + + @Override + public void validate(final String value, final Map, Object> settings) { + if (value == null || value.isEmpty()) { + throw new IllegalArgumentException( + "Setting " + remoteStoreRepositorySettingKey + " should be provided with non-empty repository ID" + ); + } else { + validateRemoteStoreSettingEnabled(settings, remoteStoreRepositorySettingKey); } - }, + } + + @Override + public Iterator> settings() { + final List> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING); + return settings.iterator(); + } + } + + /** + * Used to specify remote store data repository to use for this index. + */ + public static final Setting INDEX_REMOTE_SEGMENT_STORE_DATA_REPOSITORY_SETTING = Setting.simpleString( + SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, + new RepositoryNameValidator(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY), + Property.IndexScope, + Property.PrivateIndex, + Property.Dynamic + ); + + /** + * Used to specify remote store metadata repository to use for this index. + */ + public static final Setting INDEX_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY_SETTING = Setting.simpleString( + SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY, + new RepositoryNameValidator(SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY), Property.IndexScope, Property.PrivateIndex, Property.Dynamic ); - private static void validateRemoteStoreSettingEnabled(final Map, Object> settings, Setting setting) { + private static void validateRemoteStoreSettingEnabled(final Map, Object> settings, String settingKey) { final Boolean isRemoteSegmentStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING); if (isRemoteSegmentStoreEnabled == false) { throw new IllegalArgumentException( "Settings " - + setting.getKey() + + settingKey + " can only be set/enabled when " + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey() + " is set to true" @@ -409,39 +428,9 @@ private static void validateRemoteStoreSettingEnabled(final Map, Obje } } - public static final Setting INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING = Setting.simpleString( - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, - new Setting.Validator<>() { - - @Override - public void validate(final String value) {} - - @Override - public void validate(final String value, final Map, Object> settings) { - if (value == null || value.isEmpty()) { - throw new IllegalArgumentException( - "Setting " + INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey() + " should be provided with non-empty repository ID" - ); - } else { - final Boolean isRemoteTranslogStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING); - if (isRemoteTranslogStoreEnabled == null || isRemoteTranslogStoreEnabled == false) { - throw new IllegalArgumentException( - "Settings " - + INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey() - + " can only be set/enabled when " - + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey() - + " is set to true" - ); - } - } - } - - @Override - public Iterator> settings() { - final List> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING); - return settings.iterator(); - } - }, + public static final Setting INDEX_REMOTE_TRANSLOG_DATA_REPOSITORY_SETTING = Setting.simpleString( + SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, + new RepositoryNameValidator(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY), Property.IndexScope, Property.PrivateIndex, Property.Dynamic diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index a180876685f9c..e32f5664755c3 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -136,9 +136,10 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING; import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; @@ -1027,19 +1028,26 @@ public static void updateReplicationStrategy( */ public static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, Settings nodeSettings) { if (RemoteStoreNodeAttribute.isRemoteStoreAttributePresent(nodeSettings)) { + String segmentDataRepository = nodeSettings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY + ); settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, segmentDataRepository) .put( - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, - nodeSettings.get( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY - ) - ) - .put( - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, + + SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, nodeSettings.get( - Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY ) ); + String segmentMetadataRepository = nodeSettings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY + ); + if (segmentMetadataRepository != null) { + settingsBuilder.put(SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY, segmentMetadataRepository); + } else { + settingsBuilder.put(SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY, segmentDataRepository); + } } } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index c6c312d6b6eea..d5d43939a9147 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -228,8 +228,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // Settings for remote store enablement IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING, - IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING, - IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING, + IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_DATA_REPOSITORY_SETTING, + IndexMetadata.INDEX_REMOTE_TRANSLOG_DATA_REPOSITORY_SETTING, IndexSettings.INDEX_DOC_ID_FUZZY_SET_ENABLED_SETTING, IndexSettings.INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING, diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index a7b29314210df..97419d3e3e553 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -508,7 +508,8 @@ public synchronized IndexShard createShard( seedRemote = true; } remoteDirectory = ((RemoteSegmentStoreDirectoryFactory) remoteDirectoryFactory).newDirectory( - RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(this.indexSettings.getNodeSettings()), + RemoteStoreNodeAttribute.getRemoteStoreSegmentDataRepo(this.indexSettings.getNodeSettings()), + RemoteStoreNodeAttribute.getRemoteStoreSegmentMetadataRepo(this.indexSettings.getNodeSettings()), this.indexSettings.getUUID(), shardId, this.indexSettings.getRemoteStorePathStrategy() diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 388de65ca58a1..96206d3ed8235 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -729,8 +729,9 @@ public static IndexMergePolicy fromString(String text) { private final ReplicationType replicationType; private final boolean isRemoteStoreEnabled; private volatile TimeValue remoteTranslogUploadBufferInterval; - private final String remoteStoreTranslogRepository; - private final String remoteStoreRepository; + private final String remoteStoreTranslogDataRepository; + private final String remoteSegmentStoreDataRepository; + private final String remoteSegmentStoreMetadataRepository; private final boolean isRemoteSnapshot; private int remoteTranslogKeepExtraGen; private Version extendedCompatibilitySnapshotVersion; @@ -917,9 +918,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); replicationType = IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(settings); isRemoteStoreEnabled = settings.getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false); - remoteStoreTranslogRepository = settings.get(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY); + remoteStoreTranslogDataRepository = settings.get(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY); remoteTranslogUploadBufferInterval = INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings); - remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY); + remoteSegmentStoreDataRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY); + remoteSegmentStoreMetadataRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY); this.remoteTranslogKeepExtraGen = INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.get(settings); isRemoteSnapshot = IndexModule.Type.REMOTE_SNAPSHOT.match(this.settings); @@ -1266,12 +1268,16 @@ public boolean isRemoteTranslogStoreEnabled() { /** * Returns if remote store is enabled for this index. */ - public String getRemoteStoreRepository() { - return remoteStoreRepository; + public String getRemoteSegmentStoreDataRepository() { + return remoteSegmentStoreDataRepository; } - public String getRemoteStoreTranslogRepository() { - return remoteStoreTranslogRepository; + public String getRemoteSegmentStoreMetadataRepository() { + return remoteSegmentStoreMetadataRepository; + } + + public String getRemoteStoreTranslogDataRepository() { + return remoteStoreTranslogDataRepository; } /** diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java index f5e342d28fde1..d44d9000741a4 100644 --- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java @@ -407,6 +407,7 @@ void recoverFromSnapshotAndRemoteStore( threadPool ); RemoteSegmentStoreDirectory sourceRemoteDirectory = (RemoteSegmentStoreDirectory) directoryFactory.newDirectory( + remoteStoreRepository, remoteStoreRepository, indexUUID, shardId, diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index ec1163fe91b6c..2310e3d95fcae 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -903,6 +903,7 @@ public static void remoteDirectoryCleanup( ) { try { RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = (RemoteSegmentStoreDirectory) remoteDirectoryFactory.newDirectory( + remoteStoreRepoForIndex, remoteStoreRepoForIndex, indexUUID, shardId, diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index e462f6d4ac011..da3ccfc882a4f 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -50,23 +50,37 @@ public RemoteSegmentStoreDirectoryFactory(Supplier reposito @Override public Directory newDirectory(IndexSettings indexSettings, ShardPath path) throws IOException { - String repositoryName = indexSettings.getRemoteStoreRepository(); + String dataRepositoryName = indexSettings.getRemoteSegmentStoreDataRepository(); + String metadataRepositoryName = indexSettings.getRemoteSegmentStoreMetadataRepository(); String indexUUID = indexSettings.getIndex().getUUID(); - return newDirectory(repositoryName, indexUUID, path.getShardId(), indexSettings.getRemoteStorePathStrategy()); + return newDirectory( + dataRepositoryName, + metadataRepositoryName, + indexUUID, + path.getShardId(), + indexSettings.getRemoteStorePathStrategy() + ); } - public Directory newDirectory(String repositoryName, String indexUUID, ShardId shardId, RemoteStorePathStrategy pathStrategy) - throws IOException { + public Directory newDirectory( + String dataRepositoryName, + String metadataRepositoryName, + String indexUUID, + ShardId shardId, + RemoteStorePathStrategy pathStrategy + ) throws IOException { assert Objects.nonNull(pathStrategy); - try (Repository repository = repositoriesService.get().repository(repositoryName)) { - - assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; - BlobStoreRepository blobStoreRepository = ((BlobStoreRepository) repository); - BlobPath repositoryBasePath = blobStoreRepository.basePath(); + try ( + Repository dataRepository = repositoriesService.get().repository(dataRepositoryName); + Repository metadataRepository = repositoriesService.get().repository(metadataRepositoryName) + ) { + assert dataRepository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; + BlobStoreRepository blobStoreDataRepository = ((BlobStoreRepository) dataRepository); + BlobPath dataRepositoryBasePath = blobStoreDataRepository.basePath(); String shardIdStr = String.valueOf(shardId.id()); RemoteStorePathStrategy.PathInput dataPathInput = RemoteStorePathStrategy.PathInput.builder() - .basePath(repositoryBasePath) + .basePath(dataRepositoryBasePath) .indexUUID(indexUUID) .shardId(shardIdStr) .dataCategory(SEGMENTS) @@ -75,13 +89,15 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s // Derive the path for data directory of SEGMENTS BlobPath dataPath = pathStrategy.generatePath(dataPathInput); RemoteDirectory dataDirectory = new RemoteDirectory( - blobStoreRepository.blobStore().blobContainer(dataPath), - blobStoreRepository::maybeRateLimitRemoteUploadTransfers, - blobStoreRepository::maybeRateLimitRemoteDownloadTransfers + blobStoreDataRepository.blobStore().blobContainer(dataPath), + blobStoreDataRepository::maybeRateLimitRemoteUploadTransfers, + blobStoreDataRepository::maybeRateLimitRemoteDownloadTransfers ); + BlobStoreRepository blobStoreMetadataRepository = ((BlobStoreRepository) metadataRepository); + BlobPath metadataRepositoryBasePath = blobStoreMetadataRepository.basePath(); RemoteStorePathStrategy.PathInput mdPathInput = RemoteStorePathStrategy.PathInput.builder() - .basePath(repositoryBasePath) + .basePath(metadataRepositoryBasePath) .indexUUID(indexUUID) .shardId(shardIdStr) .dataCategory(SEGMENTS) @@ -89,12 +105,12 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s .build(); // Derive the path for metadata directory of SEGMENTS BlobPath mdPath = pathStrategy.generatePath(mdPathInput); - RemoteDirectory metadataDirectory = new RemoteDirectory(blobStoreRepository.blobStore().blobContainer(mdPath)); + RemoteDirectory metadataDirectory = new RemoteDirectory(blobStoreMetadataRepository.blobStore().blobContainer(mdPath)); // The path for lock is derived within the RemoteStoreLockManagerFactory RemoteStoreLockManager mdLockManager = RemoteStoreLockManagerFactory.newLockManager( repositoriesService.get(), - repositoryName, + dataRepositoryName, indexUUID, shardIdStr, pathStrategy diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 7e2ea5a77cbfa..b7831cd777d9b 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -537,7 +537,7 @@ private static BiFunction getTrans return new RemoteBlobStoreInternalTranslogFactory( repositoriesServiceSupplier, threadPool, - indexSettings.getRemoteStoreTranslogRepository(), + indexSettings.getRemoteStoreTranslogDataRepository(), remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardRouting.shardId()), remoteStoreSettings ); @@ -545,7 +545,7 @@ private static BiFunction getTrans return new RemoteBlobStoreInternalTranslogFactory( repositoriesServiceSupplier, threadPool, - RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(indexSettings.getNodeSettings()), + RemoteStoreNodeAttribute.getRemoteStoreTranslogDataRepo(indexSettings.getNodeSettings()), remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardRouting.shardId()), remoteStoreSettings ); diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index a3bfe1195d8cc..0d5ccea2b5491 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -35,8 +35,9 @@ public class RemoteStoreNodeAttribute { public static final String REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX = "remote_store"; - public static final String REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.repository"; - public static final String REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.translog.repository"; + public static final String REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.repository"; + public static final String REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.metadata_repository"; + public static final String REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.translog.repository"; public static final String REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.state.repository"; public static final String REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s.type"; public static final String REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT = "remote_store.repository.%s." @@ -143,10 +144,13 @@ private RepositoriesMetadata buildRepositoriesMetadata(DiscoveryNode node) { private Set getValidatedRepositoryNames(DiscoveryNode node) { Set repositoryNames = new HashSet<>(); - if (node.getAttributes().containsKey(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY) - || node.getAttributes().containsKey(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY)) { - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY)); - repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY)); + if (node.getAttributes().containsKey(REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY) + || node.getAttributes().containsKey(REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY)) { + repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY)); + if (node.getAttributes().get(REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY) != null) { + repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY)); + } + repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY)); repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)); } else if (node.getAttributes().containsKey(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)) { repositoryNames.add(validateAttributeNonNull(node, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)); @@ -159,8 +163,10 @@ public static boolean isRemoteStoreAttributePresent(Settings settings) { } public static boolean isRemoteDataAttributePresent(Settings settings) { - return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY).isEmpty() == false - || settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY).isEmpty() == false; + return settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY) + .isEmpty() == false + || settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY) + .isEmpty() == false; } public static boolean isRemoteClusterStateAttributePresent(Settings settings) { @@ -168,12 +174,22 @@ public static boolean isRemoteClusterStateAttributePresent(Settings settings) { .isEmpty() == false; } - public static String getRemoteStoreSegmentRepo(Settings settings) { - return settings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY); + public static String getRemoteStoreSegmentDataRepo(Settings settings) { + return settings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY + ); } - public static String getRemoteStoreTranslogRepo(Settings settings) { - return settings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY); + public static String getRemoteStoreSegmentMetadataRepo(Settings settings) { + return settings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY + ); + } + + public static String getRemoteStoreTranslogDataRepo(Settings settings) { + return settings.get( + Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY + ); } public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java index ce2ffd8bf3fb4..723fe1463b37f 100644 --- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java @@ -2702,7 +2702,7 @@ public void snapshotRemoteStoreIndexShard( indexTotalNumberOfFiles, indexTotalFileSize, store.indexSettings().getUUID(), - store.indexSettings().getRemoteStoreRepository(), + store.indexSettings().getRemoteSegmentStoreDataRepository(), this.basePath().toString(), fileNames, pathStrategy.getType(), diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java index ed73c2ef6ace5..928de1b473b51 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/remotestore/stats/TransportRemoteStoreStatsActionTests.java @@ -41,7 +41,7 @@ import org.mockito.Mockito; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.mockito.ArgumentMatchers.any; @@ -74,7 +74,7 @@ public void setUp() throws Exception { settings(Version.CURRENT).put(SETTING_INDEX_UUID, INDEX.getUUID()) .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-test-repo") + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-test-repo") .build() ) .numberOfShards(2) diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index 5eafe63e63fad..75894be8ea92d 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -72,8 +72,8 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; import static org.opensearch.test.VersionUtils.allVersions; @@ -499,8 +499,8 @@ public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentAttributesJoin .build(); for (Map.Entry nodeAttribute : existingNodeAttributes.entrySet()) { - if (nodeAttribute.getKey() != REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY - && nodeAttribute.getKey() != REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY + if (nodeAttribute.getKey() != REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY + && nodeAttribute.getKey() != REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY && nodeAttribute.getKey() != REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY) { remoteStoreNodeAttributes.put(nodeAttribute.getKey(), nodeAttribute.getValue() + "-new"); validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); @@ -523,10 +523,10 @@ public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentNameAttributes .build(); for (Map.Entry nodeAttribute : existingNodeAttributes.entrySet()) { - if (REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { + if (REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO + "new", TRANSLOG_REPO); validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); - } else if (REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { + } else if (REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO + "new"); validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); } else if (REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { @@ -955,11 +955,11 @@ private Map remoteStoreNodeAttributes(String segmentRepoName, St return new HashMap<>() { { - put(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName); + put(REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName); put(segmentRepositoryTypeAttributeKey, "s3"); put(segmentRepositorySettingsAttributeKeyPrefix + "bucket", "segment_bucket"); put(segmentRepositorySettingsAttributeKeyPrefix + "base_path", "/segment/path"); - put(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName); + put(REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName); putIfAbsent(translogRepositoryTypeAttributeKey, "s3"); putIfAbsent(translogRepositorySettingsAttributeKeyPrefix + "bucket", "translog_bucket"); putIfAbsent(translogRepositorySettingsAttributeKeyPrefix + "base_path", "/translog/path"); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index fa71b77648d35..b240fe47f450a 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -125,9 +125,9 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_READ_ONLY; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.aggregateIndexSettings; @@ -148,8 +148,8 @@ import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.indices.ShardLimitValidatorTests.createTestShardLimitService; import static org.opensearch.node.Node.NODE_ATTRIBUTES; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; @@ -169,9 +169,9 @@ public class MetadataCreateIndexServiceTests extends OpenSearchTestCase { private QueryShardContext queryShardContext; private ClusterSettings clusterSettings; private static final String segmentRepositoryNameAttributeKey = NODE_ATTRIBUTES.getKey() - + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; + + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; private static final String translogRepositoryNameAttributeKey = NODE_ATTRIBUTES.getKey() - + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; + + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; final String REPLICATION_MISMATCH_VALIDATION_ERROR = "Validation Failed: 1: index setting [index.replication.type] is not allowed to be set as [cluster.index.restrict.replication.type=true];"; @@ -1475,7 +1475,7 @@ public void testRemoteStoreDisabledByUserIndexSettings() { public void testRemoteStoreOverrideSegmentRepoIndexSettings() { final Settings.Builder requestSettings = Settings.builder(); requestSettings.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo"); + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo"); withTemporaryClusterService(((clusterService, threadPool) -> { MetadataCreateIndexService checkerService = new MetadataCreateIndexService( Settings.EMPTY, @@ -1505,7 +1505,7 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { String.format( Locale.ROOT, "private index setting [%s] can not be set explicitly", - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY + SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY ) ) ); @@ -1514,7 +1514,7 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { public void testRemoteStoreOverrideTranslogRepoIndexSettings() { final Settings.Builder requestSettings = Settings.builder(); - requestSettings.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo"); + requestSettings.put(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "my-custom-repo"); withTemporaryClusterService(((clusterService, threadPool) -> { MetadataCreateIndexService checkerService = new MetadataCreateIndexService( Settings.EMPTY, @@ -1544,7 +1544,7 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() { String.format( Locale.ROOT, "private index setting [%s] can not be set explicitly", - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY + SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY ) ) ); @@ -1614,7 +1614,7 @@ public void testRemoteCustomData() { private IndexMetadata testRemoteCustomData(boolean remoteStoreEnabled, PathType pathType) { Settings.Builder settingsBuilder = Settings.builder(); if (remoteStoreEnabled) { - settingsBuilder.put(NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "test"); + settingsBuilder.put(NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, "test"); } settingsBuilder.put(IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), pathType.toString()); Settings settings = settingsBuilder.build(); @@ -2116,8 +2116,8 @@ private void verifyRemoteStoreIndexSettings( ) { assertEquals(replicationType, indexSettings.get(SETTING_REPLICATION_TYPE)); assertEquals(isRemoteSegmentEnabled, indexSettings.get(SETTING_REMOTE_STORE_ENABLED)); - assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY)); - assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY)); + assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY)); + assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY)); assertEquals(translogBufferInterval, INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)); } diff --git a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java index c8a6fc76ce820..5607e90d28158 100644 --- a/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java +++ b/server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java @@ -87,7 +87,7 @@ public void testRolesAreSorted() { public void testRemoteStoreRedactionInToString() { final Set roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)); Map attributes = new HashMap<>(); - attributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "test-repo"); + attributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, "test-repo"); attributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, "test-repo"); final DiscoveryNode node = new DiscoveryNode( "name", diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java index 43363407d9249..cb3a50e6b1fb1 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java @@ -64,9 +64,9 @@ import java.util.Locale; import java.util.Map; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; @@ -603,8 +603,8 @@ private IndexMetadata.Builder getIndexMetadataBuilder(boolean isRemoteStoreBacke Settings.Builder builder = settings(Version.CURRENT); if (isRemoteStoreBackedIndex) { builder.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, TEST_REPO) - .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, TEST_REPO) + .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, TEST_REPO) + .put(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, TEST_REPO) .put(SETTING_REMOTE_STORE_ENABLED, true); } return IndexMetadata.builder(TEST_INDEX).settings(builder).numberOfShards(shardCount).numberOfReplicas(replicaCount); diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index 4ce4936c047d9..f080b816ef505 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -236,7 +236,7 @@ private IndexService newIndexService(IndexModule module) throws IOException { return new RemoteBlobStoreInternalTranslogFactory( repositoriesServiceReference::get, threadPool, - indexSettings.getRemoteStoreTranslogRepository(), + indexSettings.getRemoteStoreTranslogDataRepository(), new RemoteTranslogTransferTracker(shardRouting.shardId(), 10), DefaultRemoteStoreSettings.INSTANCE ); diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index 474ec73d5fe61..4769d19c61da0 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -823,7 +823,7 @@ public void testRemoteRepositoryDefaultSetting() { Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertNull(settings.getRemoteStoreRepository()); + assertNull(settings.getRemoteSegmentStoreDataRepository()); } public void testRemoteRepositoryExplicitSetting() { @@ -833,28 +833,28 @@ public void testRemoteRepositoryExplicitSetting() { .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "repo1") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "repo1") .build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertEquals("repo1", settings.getRemoteStoreRepository()); + assertEquals("repo1", settings.getRemoteSegmentStoreDataRepository()); } public void testSetRemoteRepositoryFailsWhenRemoteStoreIsNotEnabled() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "repo1") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "repo1") .build(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, - () -> IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(indexSettings) + () -> IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_DATA_REPOSITORY_SETTING.get(indexSettings) ); assertEquals( String.format( Locale.ROOT, "Settings %s can only be set/enabled when %s is set to true", - IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, IndexMetadata.SETTING_REMOTE_STORE_ENABLED ), iae.getMessage() @@ -865,17 +865,17 @@ public void testSetRemoteRepositoryFailsWhenEmptyString() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "") .build(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, - () -> IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(indexSettings) + () -> IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_DATA_REPOSITORY_SETTING.get(indexSettings) ); assertEquals( String.format( Locale.ROOT, "Setting %s should be provided with non-empty repository ID", - IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY + IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY ), iae.getMessage() ); @@ -887,7 +887,7 @@ public void testRemoteTranslogRepoDefaultSetting() { Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertNull(settings.getRemoteStoreRepository()); + assertNull(settings.getRemoteSegmentStoreDataRepository()); } public void testRemoteTranslogExplicitSetting() { @@ -895,13 +895,13 @@ public void testRemoteTranslogExplicitSetting() { "index", Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "tlog-store") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "tlog-store") .put(IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "200ms") .build() ); IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY); - assertNull(settings.getRemoteStoreRepository()); - assertEquals("tlog-store", settings.getRemoteStoreTranslogRepository()); + assertNull(settings.getRemoteSegmentStoreDataRepository()); + assertEquals("tlog-store", settings.getRemoteStoreTranslogDataRepository()); assertEquals(TimeValue.timeValueMillis(200), settings.getRemoteTranslogUploadBufferInterval()); } @@ -909,11 +909,11 @@ public void testSetRemoteTranslogRepositoryFailsWhenRemoteStoreIsNotEnabled() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "repo1") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "repo1") .build(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, - () -> IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(indexSettings) + () -> IndexMetadata.INDEX_REMOTE_TRANSLOG_DATA_REPOSITORY_SETTING.get(indexSettings) ); assertEquals( "Settings index.remote_store.translog.repository can only be set/enabled when index.remote_store.enabled is set to true", @@ -925,11 +925,11 @@ public void testSetRemoteTranslogRepositoryFailsWhenEmptyString() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "") .build(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, - () -> IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(indexSettings) + () -> IndexMetadata.INDEX_REMOTE_TRANSLOG_DATA_REPOSITORY_SETTING.get(indexSettings) ); assertEquals("Setting index.remote_store.translog.repository should be provided with non-empty repository ID", iae.getMessage()); } diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index e5bfa8caee79a..2887c0faf40ab 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -2733,8 +2733,8 @@ public void testRelocatedForRemoteTranslogBackedIndexWithAsyncDurability() throw .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "seg-test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "txlog-test") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "seg-test") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "txlog-test") .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC) .build(); final IndexShard indexShard = newStartedShard(true, settings, new NRTReplicationEngineFactory()); @@ -2845,8 +2845,8 @@ public void testSyncSegmentsFromGivenRemoteSegmentStore() throws IOException { Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStorePath + "__test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStorePath + "__test") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, remoteStorePath + "__test") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, remoteStorePath + "__test") .build(), new InternalEngineFactory() ); @@ -2886,8 +2886,8 @@ public void testSyncSegmentsFromGivenRemoteSegmentStore() throws IOException { Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStorePath + "__test1") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStorePath + "__test1") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, remoteStorePath + "__test1") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, remoteStorePath + "__test1") .build(), new InternalEngineFactory() ); @@ -2929,8 +2929,8 @@ public void testRestoreShardFromRemoteStore(boolean performFlush) throws IOExcep Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStorePath + "__test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, remoteStorePath + "__test") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, remoteStorePath + "__test") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, remoteStorePath + "__test") .build(), new InternalEngineFactory() ); @@ -4821,8 +4821,8 @@ public void testTranslogFactoryForRemoteTranslogBackedPrimaryShard() throws IOEx .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "seg-test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "txlog-test") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "seg-test") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "txlog-test") .build(); final IndexShard primaryShard = newStartedShard(true, primarySettings, new NRTReplicationEngineFactory()); assertEquals(primaryShard.getEngine().getClass(), InternalEngine.class); @@ -4837,8 +4837,8 @@ public void testTranslogFactoryForRemoteTranslogBackedReplicaShard() throws IOEx .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "seg-test") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "txlog-test") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "seg-test") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "txlog-test") .build(); final IndexShard replicaShard = newStartedShard(false, primarySettings, new NRTReplicationEngineFactory()); assertEquals(replicaShard.getEngine().getClass(), NRTReplicationEngine.class); diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java index 57a561bc8f2a3..9acdc7bcb616d 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteIndexShardTests.java @@ -64,8 +64,8 @@ public class RemoteIndexShardTests extends SegmentReplicationIndexShardTests { private static final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, REPOSITORY_NAME) - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, REPOSITORY_NAME) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, REPOSITORY_NAME) .build(); protected Settings getIndexSettings() { diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java index 33f6c67b94b3d..da1fe577c182f 100644 --- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java +++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java @@ -70,8 +70,8 @@ public void setup(boolean primary, int numberOfDocs) throws IOException { primary, Settings.builder() .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "temp-fs") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "temp-fs") .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(), new InternalEngineFactory() @@ -537,8 +537,8 @@ private Tuple mockIn true, Settings.builder() .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "temp-fs") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "temp-fs") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "temp-fs") .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(), new InternalEngineFactory() diff --git a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java index 85864eebd6d0d..80ae0ccee4c0a 100644 --- a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java @@ -37,7 +37,7 @@ public class ReplicaRecoveryWithRemoteTranslogOnPrimaryTests extends OpenSearchI private static final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "translog-repo") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "translog-repo") .put(IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "100ms") .build(); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java index cad5e47531cc6..58e5eb1c45ba6 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java @@ -63,7 +63,7 @@ public void setup() { public void testNewDirectory() throws IOException { Settings settings = Settings.builder() .put(IndexMetadata.SETTING_INDEX_UUID, "uuid_1") - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "remote_store_repository") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "remote_store_repository") .build(); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings); Path tempDir = createTempDir().resolve(indexSettings.getUUID()).resolve("0"); @@ -109,7 +109,7 @@ public void testNewDirectory() throws IOException { public void testNewDirectoryRepositoryDoesNotExist() { Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "remote_store_repository") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "remote_store_repository") .build(); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings); Path tempDir = createTempDir().resolve(indexSettings.getUUID()).resolve("0"); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 44ddd2de9d007..5da560dc4f350 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -146,6 +146,7 @@ public void setup() throws IOException { remoteDataDirectory = mock(RemoteDirectory.class); remoteMetadataDirectory = mock(RemoteDirectory.class); mdLockManager = mock(RemoteStoreMetadataLockManager.class); + threadPool = mock(ThreadPool.class); testUploadTracker = new TestUploadListener(); @@ -700,7 +701,7 @@ public void testCleanupAsync() throws Exception { threadPool, indexShard.shardId() ); - when(remoteSegmentStoreDirectoryFactory.newDirectory(any(), any(), any(), any())).thenReturn(remoteSegmentDirectory); + when(remoteSegmentStoreDirectoryFactory.newDirectory(any(), any(), any(), any(), any())).thenReturn(remoteSegmentDirectory); String repositoryName = "test-repository"; String indexUUID = "test-idx-uuid"; ShardId shardId = new ShardId(Index.UNKNOWN_INDEX_NAME, indexUUID, Integer.parseInt("0")); @@ -716,7 +717,7 @@ public void testCleanupAsync() throws Exception { shardId, pathStrategy ); - verify(remoteSegmentStoreDirectoryFactory).newDirectory(repositoryName, indexUUID, shardId, pathStrategy); + verify(remoteSegmentStoreDirectoryFactory).newDirectory(repositoryName, repositoryName, indexUUID, shardId, pathStrategy); verify(threadPool, times(0)).executor(ThreadPool.Names.REMOTE_PURGE); verify(remoteMetadataDirectory).delete(); verify(remoteDataDirectory).delete(); diff --git a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java index 131514eb019b3..beaf6b071cfe1 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RemoteStorePeerRecoverySourceHandlerTests.java @@ -25,7 +25,7 @@ public class RemoteStorePeerRecoverySourceHandlerTests extends OpenSearchIndexLe private static final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "translog-repo") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "translog-repo") .put(IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey(), "100ms") .build(); diff --git a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java index 287962b158c79..c619fc6c401e2 100644 --- a/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java @@ -42,8 +42,8 @@ public class RemoteStoreReplicationSourceTests extends OpenSearchIndexLevelRepli private IndexShard replicaShard; private final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, true) - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-repo") - .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-repo") + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-repo") + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "my-repo") .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); diff --git a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java index c4ba271d27ae9..7bdb103f226d0 100644 --- a/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java +++ b/server/src/test/java/org/opensearch/node/RemoteStoreNodeAttributeTests.java @@ -28,8 +28,8 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_CRYPTO_SETTINGS_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; public class RemoteStoreNodeAttributeTests extends OpenSearchTestCase { @@ -43,9 +43,9 @@ public void testCryptoMetadata() throws UnknownHostException { String repoCryptoMetadataKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, repoName); String repoCryptoMetadataSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_CRYPTO_SETTINGS_PREFIX, repoName); Map attr = Map.of( - REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, - REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, @@ -89,9 +89,9 @@ public void testInvalidCryptoMetadata() throws UnknownHostException { String repoCryptoMetadataKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, repoName); String repoCryptoMetadataSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_CRYPTO_SETTINGS_PREFIX, repoName); Map attr = Map.of( - REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, - REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, @@ -122,9 +122,9 @@ public void testNoCryptoMetadata() throws UnknownHostException { String repoTypeSettingKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, repoName); String repoSettingsKey = String.format(Locale.ROOT, REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, repoName); Map attr = Map.of( - REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, - REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName, diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index 9cca495cced72..5e1940d1769fc 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -62,8 +62,8 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.hamcrest.Matchers.equalTo; /** @@ -97,10 +97,10 @@ private Settings buildRemoteStoreNodeAttributes(String repoName, Path repoPath) ); return Settings.builder() - .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) + .put("node.attr." + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) .put(repoTypeAttributeKey, FsRepository.TYPE) .put(repoSettingsAttributeKeyPrefix + "location", repoPath) - .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) + .put("node.attr." + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) .put(repoTypeAttributeKey, FsRepository.TYPE) .put(repoSettingsAttributeKeyPrefix + "location", repoPath) .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 4dd4c734a1701..219111d830bfa 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -658,7 +658,7 @@ protected IndexShard newShard( RepositoriesService mockRepoSvc = mock(RepositoriesService.class); if (indexSettings.isRemoteStoreEnabled() || indexSettings.isAssignedOnRemoteNode()) { - String remoteStoreRepository = indexSettings.getRemoteStoreRepository(); + String remoteStoreRepository = indexSettings.getRemoteSegmentStoreDataRepository(); // remote path via setting a repository . This is a hack used for shards are created using reset . // since we can't get remote path from IndexShard directly, we are using repository to store it . if (remoteStoreRepository != null && remoteStoreRepository.endsWith("__test")) { @@ -681,7 +681,7 @@ protected IndexShard newShard( return new RemoteBlobStoreInternalTranslogFactory( () -> mockRepoSvc, threadPool, - settings.getRemoteStoreTranslogRepository(), + settings.getRemoteStoreTranslogDataRepository(), new RemoteTranslogTransferTracker(shardRouting.shardId(), 20), DefaultRemoteStoreSettings.INSTANCE ); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index c26c3f8d21380..5a5b1aa6547a1 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -216,8 +216,8 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.test.XContentTestUtils.convertToMap; import static org.opensearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -2605,10 +2605,10 @@ private static Settings buildRemoteStoreNodeAttributes( ); Settings.Builder settings = Settings.builder() - .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) + .put("node.attr." + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) .put(segmentRepoTypeAttributeKey, segmentRepoType) .put(segmentRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) - .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) + .put("node.attr." + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) .put(translogRepoTypeAttributeKey, translogRepoType) .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath) .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) From 9a0822a4f66d595daef89507fc4791013b2e12ca Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 3 Apr 2024 13:18:37 +0530 Subject: [PATCH 2/8] Add support to integ tests to pass separate metadata repo Signed-off-by: Sachin Kale --- .../RemoteStoreBaseIntegTestCase.java | 24 +++++- .../test/OpenSearchIntegTestCase.java | 83 +++++++++++++++---- 2 files changed, 86 insertions(+), 21 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index d7ad0daa43524..c3a6a9f533679 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -63,6 +63,7 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected static final String SEGMENT_METADATA_REPOSITORY_NAME = "test-remote-segment-metadata-repo"; protected static final String REPOSITORY_2_NAME = "test-remote-store-repo-2"; protected static final int SHARD_COUNT = 1; protected static int REPLICA_COUNT = 1; @@ -72,6 +73,7 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase { protected static final String MAX_SEQ_NO_REFRESHED_OR_FLUSHED = "max-seq-no-refreshed-or-flushed"; protected Path segmentRepoPath; + protected Path segmentMetadataRepoPath; protected Path translogRepoPath; protected boolean clusterSettingsSuppliedByTest = false; private final List documentKeys = List.of( @@ -133,15 +135,29 @@ protected Map indexData(int numberOfIterations, boolean invokeFlus protected Settings nodeSettings(int nodeOrdinal) { if (segmentRepoPath == null || translogRepoPath == null) { segmentRepoPath = randomRepoPath().toAbsolutePath(); + segmentMetadataRepoPath = randomRepoPath().toAbsolutePath(); translogRepoPath = randomRepoPath().toAbsolutePath(); } if (clusterSettingsSuppliedByTest) { return Settings.builder().put(super.nodeSettings(nodeOrdinal)).build(); } else { - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, translogRepoPath)) - .build(); + Settings.Builder settingsBuilder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); + if (randomBoolean()) { + settingsBuilder.put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, translogRepoPath)); + } else { + // Use a separate segment metadata repository + settingsBuilder.put( + remoteStoreClusterSettings( + REPOSITORY_NAME, + segmentRepoPath, + SEGMENT_METADATA_REPOSITORY_NAME, + segmentMetadataRepoPath, + REPOSITORY_2_NAME, + translogRepoPath + ) + ); + } + return settingsBuilder.build(); } } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 5a5b1aa6547a1..2e47b79efc557 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -217,6 +217,7 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.test.XContentTestUtils.convertToMap; import static org.opensearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder; @@ -2526,6 +2527,9 @@ public static Settings remoteStoreClusterSettings( segmentRepoName, segmentRepoPath, segmentRepoType, + null, + null, + null, translogRepoName, translogRepoPath, translogRepoType, @@ -2546,6 +2550,28 @@ public static Settings remoteStoreClusterSettings( return settingsBuilder.build(); } + public static Settings remoteStoreClusterSettings( + String segmentRepoName, + Path segmentRepoPath, + String segmentMetadataRepoName, + Path segmentMetadataRepoPath, + String translogRepoName, + Path translogRepoPath + ) { + return buildRemoteStoreNodeAttributes( + segmentRepoName, + segmentRepoPath, + ReloadableFsRepository.TYPE, + segmentMetadataRepoName, + segmentMetadataRepoPath, + ReloadableFsRepository.TYPE, + translogRepoName, + translogRepoPath, + ReloadableFsRepository.TYPE, + false + ); + } + public static Settings buildRemoteStoreNodeAttributes( String segmentRepoName, Path segmentRepoPath, @@ -2557,6 +2583,9 @@ public static Settings buildRemoteStoreNodeAttributes( segmentRepoName, segmentRepoPath, ReloadableFsRepository.TYPE, + null, + null, + null, translogRepoName, translogRepoPath, ReloadableFsRepository.TYPE, @@ -2565,23 +2594,26 @@ public static Settings buildRemoteStoreNodeAttributes( } private static Settings buildRemoteStoreNodeAttributes( - String segmentRepoName, - Path segmentRepoPath, - String segmentRepoType, + String segmentDataRepoName, + Path segmentDataRepoPath, + String segmentDataRepoType, + String segmentMetadataRepoName, + Path segmentMetadataRepoPath, + String segmentMetadataRepoType, String translogRepoName, Path translogRepoPath, String translogRepoType, boolean withRateLimiterAttributes ) { - String segmentRepoTypeAttributeKey = String.format( + String segmentDataRepoTypeAttributeKey = String.format( Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - segmentRepoName + segmentDataRepoName ); - String segmentRepoSettingsAttributeKeyPrefix = String.format( + String segmentDataRepoSettingsAttributeKeyPrefix = String.format( Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - segmentRepoName + segmentDataRepoName ); String translogRepoTypeAttributeKey = String.format( Locale.getDefault(), @@ -2596,28 +2628,45 @@ private static Settings buildRemoteStoreNodeAttributes( String stateRepoTypeAttributeKey = String.format( Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - segmentRepoName + segmentDataRepoName ); String stateRepoSettingsAttributeKeyPrefix = String.format( Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - segmentRepoName + segmentDataRepoName ); Settings.Builder settings = Settings.builder() - .put("node.attr." + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) - .put(segmentRepoTypeAttributeKey, segmentRepoType) - .put(segmentRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) + .put("node.attr." + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentDataRepoName) + .put(segmentDataRepoTypeAttributeKey, segmentDataRepoType) + .put(segmentDataRepoSettingsAttributeKeyPrefix + "location", segmentDataRepoPath) .put("node.attr." + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) .put(translogRepoTypeAttributeKey, translogRepoType) .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath) - .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) - .put(stateRepoTypeAttributeKey, segmentRepoType) - .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath); + .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentDataRepoName) + .put(stateRepoTypeAttributeKey, segmentDataRepoType) + .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentDataRepoPath); + + String segmentMetadataRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + segmentMetadataRepoName + ); + String segmentMetadataRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + segmentMetadataRepoName + ); + if (segmentMetadataRepoName != null && segmentMetadataRepoPath != null && segmentMetadataRepoType != null) { + Settings.builder() + .put("node.attr." + REMOTE_STORE_SEGMENT_METADATA_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentMetadataRepoName) + .put(segmentMetadataRepoTypeAttributeKey, segmentMetadataRepoType) + .put(segmentMetadataRepoSettingsAttributeKeyPrefix + "location", segmentMetadataRepoPath); + } if (withRateLimiterAttributes) { - settings.put(segmentRepoSettingsAttributeKeyPrefix + "compress", randomBoolean()) - .put(segmentRepoSettingsAttributeKeyPrefix + "chunk_size", 200, ByteSizeUnit.BYTES); + settings.put(segmentDataRepoSettingsAttributeKeyPrefix + "compress", randomBoolean()) + .put(segmentDataRepoSettingsAttributeKeyPrefix + "chunk_size", 200, ByteSizeUnit.BYTES); } settings.put(CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), randomFrom(PathType.values())); return settings.build(); From ac165ded87c530e22964fa97fd67874675062dea Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 3 Apr 2024 18:06:48 +0530 Subject: [PATCH 3/8] Add changelog entry Signed-off-by: Sachin Kale --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8b3e10a37a37..9683ef661addd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Reject Resize index requests (i.e, split, shrink and clone), While DocRep to SegRep migration is in progress.([#12686](https://github.com/opensearch-project/OpenSearch/pull/12686)) - Add support for more than one protocol for transport ([#12967](https://github.com/opensearch-project/OpenSearch/pull/12967)) - Add changes for overriding remote store and replication settings during snapshot restore. ([#11868](https://github.com/opensearch-project/OpenSearch/pull/11868)) +- [Remote Store] Add support to provide separate segment metadata repository ([#12993](https://github.com/opensearch-project/OpenSearch/pull/12993)) ### Dependencies - Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896)) From 44573c62ea03721252dca0d808cd8b0dde43b3a5 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 3 Apr 2024 18:57:52 +0530 Subject: [PATCH 4/8] Fix compile error Signed-off-by: Sachin Kale --- .../java/org/opensearch/index/shard/IndexShardTestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestUtils.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestUtils.java index d3a4a95c3bdef..e1b944175a487 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestUtils.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestUtils.java @@ -34,7 +34,7 @@ public static DiscoveryNode getFakeDiscoNode(String id) { public static DiscoveryNode getFakeRemoteEnabledNode(String id) { Map remoteNodeAttributes = new HashMap(); - remoteNodeAttributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "test-repo"); + remoteNodeAttributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, "test-repo"); return new DiscoveryNode( id, id, From 06b122266a578fce3a83c77ce658aa9c5060e63c Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Thu, 4 Apr 2024 10:32:05 +0530 Subject: [PATCH 5/8] Fix null metadata repo Signed-off-by: Sachin Kale --- .../index/store/RemoteSegmentStoreDirectoryFactory.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index da3ccfc882a4f..3790c8d40406a 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -70,6 +70,9 @@ public Directory newDirectory( RemoteStorePathStrategy pathStrategy ) throws IOException { assert Objects.nonNull(pathStrategy); + if (metadataRepositoryName == null) { + metadataRepositoryName = dataRepositoryName; + } try ( Repository dataRepository = repositoriesService.get().repository(dataRepositoryName); Repository metadataRepository = repositoriesService.get().repository(metadataRepositoryName) From 7d162068191e036b9eb936721cd4b184de50a674 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Thu, 4 Apr 2024 10:53:40 +0530 Subject: [PATCH 6/8] Add entry of the metadata repository setting into built in settings Signed-off-by: Sachin Kale --- .../java/org/opensearch/common/settings/IndexScopedSettings.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index d5d43939a9147..c4f32f4124be9 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -229,6 +229,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // Settings for remote store enablement IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING, IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_DATA_REPOSITORY_SETTING, + IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY_SETTING, IndexMetadata.INDEX_REMOTE_TRANSLOG_DATA_REPOSITORY_SETTING, IndexSettings.INDEX_DOC_ID_FUZZY_SET_ENABLED_SETTING, From 13917374295dafd6a6b0778d8990c57cf65e4de0 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Thu, 4 Apr 2024 12:20:10 +0530 Subject: [PATCH 7/8] Fix unit test Signed-off-by: Sachin Kale --- .../index/store/RemoteSegmentStoreDirectoryFactoryTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java index 58e5eb1c45ba6..d043d936dafc2 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactoryTests.java @@ -103,7 +103,7 @@ public void testNewDirectory() throws IOException { eq(BlobContainer.BlobNameSortOrder.LEXICOGRAPHIC), any() ); - verify(repositoriesService, times(2)).repository("remote_store_repository"); + verify(repositoriesService, times(3)).repository("remote_store_repository"); } } From 95d18c7b0998dceda73da1738a86d4861369b523 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Fri, 12 Apr 2024 11:00:57 +0530 Subject: [PATCH 8/8] Address PR comments Signed-off-by: Sachin Kale --- .../opensearch/remotestore/RemoteRestoreSnapshotIT.java | 1 - .../index/store/RemoteSegmentStoreDirectoryFactory.java | 2 +- .../java/org/opensearch/snapshots/RestoreService.java | 8 ++++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 297f3816ba395..3fc80eebaf3cc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -54,7 +54,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java index 3790c8d40406a..963b4e84e60c7 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java @@ -113,7 +113,7 @@ public Directory newDirectory( // The path for lock is derived within the RemoteStoreLockManagerFactory RemoteStoreLockManager mdLockManager = RemoteStoreLockManagerFactory.newLockManager( repositoriesService.get(), - dataRepositoryName, + metadataRepositoryName, indexUUID, shardIdStr, pathStrategy diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 980610d337a63..6062e133a6dc9 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -118,9 +118,9 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED; import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY; @@ -165,8 +165,8 @@ public class RestoreService implements ClusterStateApplier { SETTING_CREATION_DATE, SETTING_HISTORY_UUID, SETTING_REMOTE_STORE_ENABLED, - SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, - SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY + SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, + SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY ) );