From ab9b729249a22e4be4ebbc06b4d95cf24dd4e04c Mon Sep 17 00:00:00 2001 From: Ashish Singh Date: Thu, 23 May 2024 16:08:38 +0530 Subject: [PATCH] Add period at the start of the hashed prefix Signed-off-by: Ashish Singh --- .../remotestore/RemoteRestoreSnapshotIT.java | 6 +- .../RemoteStoreUploadIndexPathIT.java | 7 +- .../metadata/MetadataCreateIndexService.java | 46 ++++--- .../RemoteStoreCustomMetadataResolver.java | 31 ++++- .../index/remote/RemoteStoreEnums.java | 4 +- .../indices/RemoteStoreSettings.java | 2 +- .../MetadataCreateIndexServiceTests.java | 7 ++ .../index/remote/RemoteIndexPathTests.java | 6 +- ...emoteStoreCustomMetadataResolverTests.java | 114 ++++++++++++++++-- .../index/remote/RemoteStoreEnumsTests.java | 110 ++++++++++++++--- 10 files changed, 279 insertions(+), 54 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index f8e5079b01a36..a122b8459f1e1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -55,6 +55,7 @@ import java.util.stream.Stream; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; +import static org.opensearch.index.remote.RemoteStoreCustomMetadataResolver.PATH_TYPE_NODE_ATTR_KEY; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; @@ -215,8 +216,9 @@ public void testRestoreOperationsShallowCopyEnabled() throws Exception { * on snapshot restore. */ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() { - String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(); - internalCluster().startDataOnlyNode(); + Settings pathTypeSetting = Settings.builder().put(PATH_TYPE_NODE_ATTR_KEY, true).build(); + String clusterManagerNode = internalCluster().startClusterManagerOnlyNode(pathTypeSetting); + internalCluster().startDataOnlyNode(pathTypeSetting); String indexName1 = "testindex1"; String indexName2 = "testindex2"; String snapshotRepoName = "test-restore-snapshot-repo"; diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreUploadIndexPathIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreUploadIndexPathIT.java index 44c02dbb6d611..3fbe641fbdfaf 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreUploadIndexPathIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreUploadIndexPathIT.java @@ -23,6 +23,7 @@ import java.util.concurrent.ExecutionException; import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING; +import static org.opensearch.index.remote.RemoteStoreCustomMetadataResolver.PATH_TYPE_NODE_ATTR_KEY; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -33,7 +34,11 @@ public class RemoteStoreUploadIndexPathIT extends RemoteStoreBaseIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true).build(); + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true) + .put(PATH_TYPE_NODE_ATTR_KEY, true) + .build(); } /** diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 16edec112f123..a5bc94dcbab7b 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -215,7 +215,13 @@ public MetadataCreateIndexService( createIndexTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.CREATE_INDEX_KEY, true); Supplier minNodeVersionSupplier = () -> clusterService.state().nodes().getMinNodeVersion(); remoteStoreCustomMetadataResolver = isRemoteDataAttributePresent(settings) - ? new RemoteStoreCustomMetadataResolver(remoteStoreSettings, minNodeVersionSupplier, repositoriesServiceSupplier, settings) + ? new RemoteStoreCustomMetadataResolver( + remoteStoreSettings, + clusterService, + minNodeVersionSupplier, + repositoriesServiceSupplier, + settings + ) : null; } @@ -256,7 +262,8 @@ public void validateIndexName(String index, ClusterState state) { /** * Validates (if this index has a dot-prefixed name) whether it follows the rules for dot-prefixed indices. - * @param index The name of the index in question + * + * @param index The name of the index in question * @param isHidden Whether or not this is a hidden index */ public boolean validateDotIndex(String index, @Nullable Boolean isHidden) { @@ -323,7 +330,7 @@ public static void validateIndexOrAliasName(String index, BiFunction validatePrivateSettingsNotExplicitlySet(Settings set /** * Validates that the configured index data path (if any) is a sub-path of the configured shared data path (if any) * - * @param settings the index configured settings + * @param settings the index configured settings * @param sharedDataPath the configured `path.shared_data` (if any) * @return a list containing validaton errors or an empty list if there aren't any errors */ diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolver.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolver.java index e8a0dda5a699e..118c0f651168a 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolver.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolver.java @@ -9,16 +9,22 @@ package org.opensearch.index.remote; import org.opensearch.Version; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.settings.Settings; import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.node.Node; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.repositories.blobstore.BlobStoreRepository; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; import java.util.function.Supplier; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo; @@ -31,18 +37,24 @@ @ExperimentalApi public class RemoteStoreCustomMetadataResolver { + public final static String PATH_TYPE_ATTRIBUTE_KEY = "optimised_remote_store_enable"; + public final static String PATH_TYPE_NODE_ATTR_KEY = Node.NODE_ATTRIBUTES.getKey() + PATH_TYPE_ATTRIBUTE_KEY; + private final RemoteStoreSettings remoteStoreSettings; + private final ClusterService clusterService; private final Supplier minNodeVersionSupplier; private final Supplier repositoriesServiceSupplier; private final Settings settings; public RemoteStoreCustomMetadataResolver( RemoteStoreSettings remoteStoreSettings, + ClusterService clusterService, Supplier minNodeVersionSupplier, Supplier repositoriesServiceSupplier, Settings settings ) { this.remoteStoreSettings = remoteStoreSettings; + this.clusterService = Objects.requireNonNull(clusterService); this.minNodeVersionSupplier = minNodeVersionSupplier; this.repositoriesServiceSupplier = repositoriesServiceSupplier; this.settings = settings; @@ -52,12 +64,29 @@ public RemoteStorePathStrategy getPathStrategy() { PathType pathType; PathHashAlgorithm pathHashAlgorithm; // Min node version check ensures that we are enabling the new prefix type only when all the nodes understand it. - pathType = Version.V_2_14_0.compareTo(minNodeVersionSupplier.get()) <= 0 ? remoteStoreSettings.getPathType() : PathType.FIXED; + pathType = isPathTypeEnabled() ? remoteStoreSettings.getPathType() : PathType.FIXED; // If the path type is fixed, hash algorithm is not applicable. pathHashAlgorithm = pathType == PathType.FIXED ? null : remoteStoreSettings.getPathHashAlgorithm(); return new RemoteStorePathStrategy(pathType, pathHashAlgorithm); } + private boolean isPathTypeEnabled() { + Map nodesMap = Collections.unmodifiableMap(clusterService.state().nodes().getNodes()); + + if (nodesMap.isEmpty()) { + return false; + } + + for (String node : nodesMap.keySet()) { + DiscoveryNode nodeDiscovery = nodesMap.get(node); + Map nodeAttributes = nodeDiscovery.getAttributes(); + if (!nodeAttributes.containsKey(PATH_TYPE_ATTRIBUTE_KEY)) { + return false; + } + } + return true; + } + public boolean isTranslogMetadataEnabled() { Repository repository; try { diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java index c1ac74724e405..ff793aefd194d 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java @@ -36,6 +36,8 @@ @ExperimentalApi public class RemoteStoreEnums { + static final char REMOTE_STORE_DEDICATED_PREFIX_START_CHAR = '.'; + /** * Categories of the data in Remote store. */ @@ -112,7 +114,7 @@ boolean requiresHashAlgorithm() { public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) { assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null"; return BlobPath.cleanPath() - .add(hashAlgorithm.hash(pathInput)) + .add(REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + hashAlgorithm.hash(pathInput)) .add(pathInput.basePath()) .add(pathInput.indexUUID()) .add(pathInput.shardId()) diff --git a/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java b/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java index 074186f64a75d..a48b121a0f3e6 100644 --- a/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java +++ b/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java @@ -74,7 +74,7 @@ public class RemoteStoreSettings { @ExperimentalApi public static final Setting CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING = new Setting<>( "cluster.remote_store.index.path.type", - RemoteStoreEnums.PathType.FIXED.toString(), + RemoteStoreEnums.PathType.HASHED_PREFIX.toString(), RemoteStoreEnums.PathType::parseString, Property.NodeScope, Property.Dynamic diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 0072649e4ca72..2f1906fad6ab3 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -74,6 +74,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.remote.RemoteStoreCustomMetadataResolver; import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.index.translog.Translog; @@ -1754,8 +1755,14 @@ private IndexMetadata testRemoteCustomData(boolean remoteStoreEnabled, PathType Metadata metadata = Metadata.builder() .transientSettings(Settings.builder().put(Metadata.DEFAULT_REPLICA_COUNT_SETTING.getKey(), 1).build()) .build(); + DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); + DiscoveryNode discoveryNode = mock(DiscoveryNode.class); + when(discoveryNode.getAttributes()).thenReturn(Map.of(RemoteStoreCustomMetadataResolver.PATH_TYPE_ATTRIBUTE_KEY, "true")); + when(discoveryNodes.getNodes()).thenReturn(Map.of("node-1", discoveryNode)); + when(discoveryNodes.getMinNodeVersion()).thenReturn(Version.V_2_15_0); ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) .metadata(metadata) + .nodes(discoveryNodes) .build(); ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); when(clusterService.getSettings()).thenReturn(settings); diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java index 8ddbd383756e7..8865d3ac63cc1 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java @@ -47,7 +47,7 @@ public void testToXContentWithSegmentRepo() throws IOException { xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); xContentBuilder.endObject(); String expected = - "{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\"9BmBinD5HYs/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/data/\",\"ExCNOD8_5ew/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/data/\",\"z8wtf0yr2l4/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/metadata/\",\"VheHVwFlExE/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/metadata/\",\"IgFKbsDeUpQ/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/lock_files/\",\"pA3gy_GZtns/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/lock_files/\"]}"; + "{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\".9BmBinD5HYs/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/data/\",\".ExCNOD8_5ew/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/data/\",\".z8wtf0yr2l4/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/metadata/\",\".VheHVwFlExE/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/metadata/\",\".IgFKbsDeUpQ/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/lock_files/\",\".pA3gy_GZtns/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/lock_files/\"]}"; assertEquals(expected, xContentBuilder.toString()); } @@ -68,7 +68,7 @@ public void testToXContentForTranslogRepoOnly() throws IOException { xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); xContentBuilder.endObject(); String expected = - "{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"]},\"paths\":[\"2EaVODaKBck/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/data/\",\"dTS2VqEOUNo/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/data/\",\"PVNKNGonmZw/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/metadata/\",\"NXmt0Y6NjA8/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/metadata/\"]}"; + "{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"]},\"paths\":[\".2EaVODaKBck/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/data/\",\".dTS2VqEOUNo/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/data/\",\".PVNKNGonmZw/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/metadata/\",\".NXmt0Y6NjA8/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/metadata/\"]}"; assertEquals(expected, xContentBuilder.toString()); } @@ -92,7 +92,7 @@ public void testToXContentForBothRepos() throws IOException { xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); xContentBuilder.endObject(); String expected = - "{\"version\":\"1\",\"index_uuid\":\"csbdqiu8a7sdnjdks\",\"shard_count\":3,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"],\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\"Cjo0F6kNjYk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/data/\",\"kpayyhxct1I/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/data/\",\"p2RlgnHeIgc/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/data/\",\"gkPIurBtB1w/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/metadata/\",\"Y4YhlbxAB1c/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/metadata/\",\"HYc8fyVPouI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/metadata/\",\"igzyZCz1ysI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/lock_files/\",\"uEluEiYmptk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/lock_files/\",\"TfAD8f06_7A/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/lock_files/\",\"QqKEpasbEGs/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/data/\",\"sNyoimoe1Bw/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/data/\",\"d4YQtONfq50/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/data/\",\"zLr4UXjK8T4/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/metadata/\",\"_s8i7ZmlXGE/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/metadata/\",\"tvtD3-k5ISg/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/metadata/\"]}"; + "{\"version\":\"1\",\"index_uuid\":\"csbdqiu8a7sdnjdks\",\"shard_count\":3,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"],\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\".Cjo0F6kNjYk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/data/\",\".kpayyhxct1I/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/data/\",\".p2RlgnHeIgc/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/data/\",\".gkPIurBtB1w/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/metadata/\",\".Y4YhlbxAB1c/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/metadata/\",\".HYc8fyVPouI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/metadata/\",\".igzyZCz1ysI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/lock_files/\",\".uEluEiYmptk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/lock_files/\",\".TfAD8f06_7A/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/lock_files/\",\".QqKEpasbEGs/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/data/\",\".sNyoimoe1Bw/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/data/\",\".d4YQtONfq50/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/data/\",\".zLr4UXjK8T4/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/metadata/\",\"._s8i7ZmlXGE/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/metadata/\",\".tvtD3-k5ISg/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/metadata/\"]}"; assertEquals(expected, xContentBuilder.toString()); } diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolverTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolverTests.java index 7e702ad3773e8..45a1c05457524 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolverTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolverTests.java @@ -9,6 +9,10 @@ package org.opensearch.index.remote; import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.blobstore.BlobStore; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -18,7 +22,14 @@ import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import static org.opensearch.index.remote.RemoteStoreCustomMetadataResolver.PATH_TYPE_ATTRIBUTE_KEY; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING; import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA; @@ -28,6 +39,21 @@ public class RemoteStoreCustomMetadataResolverTests extends OpenSearchTestCase { + private ClusterService clusterService; + private final Map nodesMap = new HashMap<>(); + + private final AtomicLong nodeCounter = new AtomicLong(); + + @Before + public void setup() { + clusterService = mock(ClusterService.class); + ClusterState clusterState = mock(ClusterState.class); + when(clusterService.state()).thenReturn(clusterState); + DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); + when(clusterState.nodes()).thenReturn(discoveryNodes); + when(discoveryNodes.getNodes()).thenReturn(nodesMap); + } + RepositoriesService repositoriesService = mock(RepositoriesService.class); Settings settings = Settings.EMPTY; @@ -37,10 +63,12 @@ public void testGetPathStrategyMinVersionOlder() { RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); RemoteStoreCustomMetadataResolver resolver = new RemoteStoreCustomMetadataResolver( remoteStoreSettings, + clusterService, () -> Version.V_2_13_0, () -> repositoriesService, settings ); + addNode(false); assertEquals(PathType.FIXED, resolver.getPathStrategy().getType()); assertNull(resolver.getPathStrategy().getHashAlgorithm()); } @@ -52,10 +80,12 @@ public void testGetPathStrategyMinVersionNewer() { RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); RemoteStoreCustomMetadataResolver resolver = new RemoteStoreCustomMetadataResolver( remoteStoreSettings, + clusterService, () -> Version.V_2_14_0, () -> repositoriesService, settings ); + addNode(true); assertEquals(pathType, resolver.getPathStrategy().getType()); if (pathType.requiresHashAlgorithm()) { assertNotNull(resolver.getPathStrategy().getHashAlgorithm()); @@ -69,8 +99,10 @@ public void testGetPathStrategyStrategy() { Settings settings = Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.FIXED).build(); ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); + addNode(true); RemoteStoreCustomMetadataResolver resolver = new RemoteStoreCustomMetadataResolver( remoteStoreSettings, + clusterService, () -> Version.V_2_14_0, () -> repositoriesService, settings @@ -84,14 +116,26 @@ public void testGetPathStrategyStrategy() { .build(); clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); - resolver = new RemoteStoreCustomMetadataResolver(remoteStoreSettings, () -> Version.V_2_14_0, () -> repositoriesService, settings); + resolver = new RemoteStoreCustomMetadataResolver( + remoteStoreSettings, + clusterService, + () -> Version.V_2_14_0, + () -> repositoriesService, + settings + ); assertEquals(PathType.FIXED, resolver.getPathStrategy().getType()); // HASHED_PREFIX type with FNV_1A_COMPOSITE settings = Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX).build(); clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); - resolver = new RemoteStoreCustomMetadataResolver(remoteStoreSettings, () -> Version.V_2_14_0, () -> repositoriesService, settings); + resolver = new RemoteStoreCustomMetadataResolver( + remoteStoreSettings, + clusterService, + () -> Version.V_2_14_0, + () -> repositoriesService, + settings + ); assertEquals(PathType.HASHED_PREFIX, resolver.getPathStrategy().getType()); assertEquals(PathHashAlgorithm.FNV_1A_COMPOSITE_1, resolver.getPathStrategy().getHashAlgorithm()); @@ -99,7 +143,13 @@ public void testGetPathStrategyStrategy() { settings = Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX).build(); clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); - resolver = new RemoteStoreCustomMetadataResolver(remoteStoreSettings, () -> Version.V_2_14_0, () -> repositoriesService, settings); + resolver = new RemoteStoreCustomMetadataResolver( + remoteStoreSettings, + clusterService, + () -> Version.V_2_14_0, + () -> repositoriesService, + settings + ); assertEquals(PathType.HASHED_PREFIX, resolver.getPathStrategy().getType()); assertEquals(PathHashAlgorithm.FNV_1A_COMPOSITE_1, resolver.getPathStrategy().getHashAlgorithm()); @@ -110,7 +160,13 @@ public void testGetPathStrategyStrategy() { .build(); clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); - resolver = new RemoteStoreCustomMetadataResolver(remoteStoreSettings, () -> Version.V_2_14_0, () -> repositoriesService, settings); + resolver = new RemoteStoreCustomMetadataResolver( + remoteStoreSettings, + clusterService, + () -> Version.V_2_14_0, + () -> repositoriesService, + settings + ); assertEquals(PathType.HASHED_PREFIX, resolver.getPathStrategy().getType()); assertEquals(PathHashAlgorithm.FNV_1A_BASE64, resolver.getPathStrategy().getHashAlgorithm()); @@ -121,7 +177,13 @@ public void testGetPathStrategyStrategy() { .build(); clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); - resolver = new RemoteStoreCustomMetadataResolver(remoteStoreSettings, () -> Version.V_2_14_0, () -> repositoriesService, settings); + resolver = new RemoteStoreCustomMetadataResolver( + remoteStoreSettings, + clusterService, + () -> Version.V_2_14_0, + () -> repositoriesService, + settings + ); assertEquals(PathType.HASHED_PREFIX, resolver.getPathStrategy().getType()); assertEquals(PathHashAlgorithm.FNV_1A_BASE64, resolver.getPathStrategy().getHashAlgorithm()); } @@ -132,14 +194,17 @@ public void testGetPathStrategyStrategyWithDynamicUpdate() { Settings settings = Settings.builder().build(); ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); + addNode(true); RemoteStoreCustomMetadataResolver resolver = new RemoteStoreCustomMetadataResolver( remoteStoreSettings, + clusterService, () -> Version.V_2_14_0, () -> repositoriesService, settings ); - assertEquals(PathType.FIXED, resolver.getPathStrategy().getType()); - assertNull(resolver.getPathStrategy().getHashAlgorithm()); + assertEquals(PathType.HASHED_PREFIX, resolver.getPathStrategy().getType()); + assertNotNull(resolver.getPathStrategy().getHashAlgorithm()); + assertEquals(PathHashAlgorithm.FNV_1A_COMPOSITE_1, resolver.getPathStrategy().getHashAlgorithm()); // Set HASHED_PREFIX with default hash algorithm clusterSettings.applySettings( @@ -148,6 +213,11 @@ public void testGetPathStrategyStrategyWithDynamicUpdate() { assertEquals(PathType.HASHED_PREFIX, resolver.getPathStrategy().getType()); assertEquals(PathHashAlgorithm.FNV_1A_COMPOSITE_1, resolver.getPathStrategy().getHashAlgorithm()); + // Set FIXED with null hash algorithm + clusterSettings.applySettings(Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.FIXED).build()); + assertEquals(PathType.FIXED, resolver.getPathStrategy().getType()); + assertNull(resolver.getPathStrategy().getHashAlgorithm()); + // Set HASHED_PREFIX with FNV_1A_BASE64 hash algorithm clusterSettings.applySettings( Settings.builder() @@ -176,6 +246,21 @@ public void testGetPathStrategyStrategyWithDynamicUpdate() { assertEquals(PathHashAlgorithm.FNV_1A_BASE64, resolver.getPathStrategy().getHashAlgorithm()); } + public void testGetPathStrategyStrategyMinVersionOlderWithoutNodes() { + Settings settings = Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(PathType.values())).build(); + ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); + RemoteStoreCustomMetadataResolver resolver = new RemoteStoreCustomMetadataResolver( + remoteStoreSettings, + clusterService, + () -> Version.V_2_14_0, + () -> repositoriesService, + settings + ); + assertEquals(PathType.FIXED, resolver.getPathStrategy().getType()); + assertNull(resolver.getPathStrategy().getHashAlgorithm()); + } + public void testTranslogMetadataAllowedTrueWithMinVersionNewer() { Settings settings = Settings.builder().put(CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), true).build(); ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); @@ -187,6 +272,7 @@ public void testTranslogMetadataAllowedTrueWithMinVersionNewer() { when(blobStoreMock.isBlobMetadataEnabled()).thenReturn(true); RemoteStoreCustomMetadataResolver resolver = new RemoteStoreCustomMetadataResolver( remoteStoreSettings, + clusterService, () -> Version.CURRENT, () -> repositoriesService, settings @@ -200,6 +286,7 @@ public void testTranslogMetadataAllowedFalseWithMinVersionNewer() { RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); RemoteStoreCustomMetadataResolver resolver = new RemoteStoreCustomMetadataResolver( remoteStoreSettings, + clusterService, () -> Version.CURRENT, () -> repositoriesService, settings @@ -213,6 +300,7 @@ public void testTranslogMetadataAllowedMinVersionOlder() { RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings); RemoteStoreCustomMetadataResolver resolver = new RemoteStoreCustomMetadataResolver( remoteStoreSettings, + clusterService, () -> Version.V_2_14_0, () -> repositoriesService, settings @@ -220,4 +308,16 @@ public void testTranslogMetadataAllowedMinVersionOlder() { assertFalse(resolver.isTranslogMetadataEnabled()); } + private void addNode(boolean hashPathTypeAttr) { + DiscoveryNode discoveryNode = mock(DiscoveryNode.class); + Map nodeAttrs; + if (hashPathTypeAttr) { + nodeAttrs = Map.of(PATH_TYPE_ATTRIBUTE_KEY, "true"); + } else { + nodeAttrs = Collections.emptyMap(); + } + when(discoveryNode.getAttributes()).thenReturn(nodeAttrs); + nodesMap.put("node-" + nodeCounter.getAndIncrement(), discoveryNode); + } + } diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java index 575b397382f24..109775e265bd3 100644 --- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java +++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java @@ -165,7 +165,15 @@ public void testGeneratePathForHashedPrefixType() { BlobPath result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); assertTrue( result.buildAsString() - .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())) + .startsWith( + String.join( + SEPARATOR, + RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) + ) ); // assert with exact value for known base path @@ -180,7 +188,7 @@ public void testGeneratePathForHashedPrefixType() { .dataType(dataType) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); - assertEquals("DgSI70IciXs/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString()); + assertEquals(".DgSI70IciXs/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString()); // Translog Metadata dataType = METADATA; @@ -194,7 +202,15 @@ public void testGeneratePathForHashedPrefixType() { result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); assertTrue( result.buildAsString() - .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())) + .startsWith( + String.join( + SEPARATOR, + RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) + ) ); // assert with exact value for known base path @@ -206,7 +222,7 @@ public void testGeneratePathForHashedPrefixType() { .dataType(dataType) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); - assertEquals("oKU5SjILiy4/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", result.buildAsString()); + assertEquals(".oKU5SjILiy4/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", result.buildAsString()); // Translog Lock files - This is a negative case where the assertion will trip. dataType = LOCK_FILES; @@ -242,7 +258,15 @@ public void testGeneratePathForHashedPrefixType() { result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); assertTrue( result.buildAsString() - .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())) + .startsWith( + String.join( + SEPARATOR, + RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) + ) ); // assert with exact value for known base path @@ -254,7 +278,7 @@ public void testGeneratePathForHashedPrefixType() { .dataType(dataType) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); - assertEquals("AUBRfCIuWdk/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString()); + assertEquals(".AUBRfCIuWdk/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString()); // Segment Metadata dataType = METADATA; @@ -268,7 +292,15 @@ public void testGeneratePathForHashedPrefixType() { result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); assertTrue( result.buildAsString() - .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())) + .startsWith( + String.join( + SEPARATOR, + RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) + ) ); // assert with exact value for known base path @@ -280,7 +312,7 @@ public void testGeneratePathForHashedPrefixType() { .dataType(dataType) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); - assertEquals("erwR-G735Uw/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", result.buildAsString()); + assertEquals(".erwR-G735Uw/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", result.buildAsString()); // Segment Lockfiles dataType = LOCK_FILES; @@ -294,7 +326,15 @@ public void testGeneratePathForHashedPrefixType() { result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); assertTrue( result.buildAsString() - .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())) + .startsWith( + String.join( + SEPARATOR, + RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) + ) ); // assert with exact value for known base path @@ -306,7 +346,7 @@ public void testGeneratePathForHashedPrefixType() { .dataType(dataType) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64); - assertEquals("KeYDIk0mJXI/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", result.buildAsString()); + assertEquals(".KeYDIk0mJXI/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", result.buildAsString()); } public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { @@ -334,7 +374,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { assertTrue( result.buildAsString() .startsWith( - String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()) + String.join( + SEPARATOR, + RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) ) ); @@ -350,7 +396,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .dataType(dataType) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); - assertEquals("D10000001001000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString()); + assertEquals(".D10000001001000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString()); // Translog Metadata dataType = METADATA; @@ -365,7 +411,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { assertTrue( result.buildAsString() .startsWith( - String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()) + String.join( + SEPARATOR, + RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) ) ); @@ -379,7 +431,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); assertEquals( - "o00101001010011/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", + ".o00101001010011/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", result.buildAsString() ); @@ -418,7 +470,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { assertTrue( result.buildAsString() .startsWith( - String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()) + String.join( + SEPARATOR, + RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) ) ); @@ -431,7 +489,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .dataType(dataType) .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); - assertEquals("A01010000000101/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString()); + assertEquals(".A01010000000101/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString()); // Segment Metadata dataType = METADATA; @@ -446,7 +504,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { assertTrue( result.buildAsString() .startsWith( - String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()) + String.join( + SEPARATOR, + RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) ) ); @@ -460,7 +524,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); assertEquals( - "e10101111000001/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", + ".e10101111000001/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", result.buildAsString() ); @@ -477,7 +541,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { assertTrue( result.buildAsString() .startsWith( - String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()) + String.join( + SEPARATOR, + RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput), + basePath, + dataCategory.getName(), + dataType.getName() + ) ) ); @@ -491,7 +561,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() { .build(); result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1); assertEquals( - "K01111001100000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", + ".K01111001100000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", result.buildAsString() ); }