Introduce remote store hash algo in customData in IndexMetadata (#12986…

…) (#13407) (cherry picked from commit 170ea7a) Signed-off-by: Ashish Singh <[email protected]> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Apr 26, 2024 · 4bea236 · 4bea236
1 parent 151a671
commit 4bea236
Show file tree

Hide file tree

Showing 26 changed files with 685 additions and 270 deletions.
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
@@ -25,7 +25,8 @@
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.index.IndexService;
 import org.opensearch.index.IndexSettings;
-import org.opensearch.index.remote.RemoteStorePathType;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.replication.common.ReplicationType;
@@ -225,7 +226,7 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
 
         indexDocuments(client, indexName1, randomIntBetween(5, 10));
         ensureGreen(indexName1);
-        validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);
+        validatePathType(indexName1, PathType.FIXED, PathHashAlgorithm.FNV_1A);
 
         logger.info("--> snapshot");
         SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1)));
@@ -242,14 +243,12 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
             .get();
         assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
         ensureGreen(restoredIndexName1version1);
-        validateRemoteStorePathType(restoredIndexName1version1, RemoteStorePathType.FIXED);
+        validatePathType(restoredIndexName1version1, PathType.FIXED, PathHashAlgorithm.FNV_1A);
 
         client(clusterManagerNode).admin()
             .cluster()
             .prepareUpdateSettings()
-            .setTransientSettings(
-                Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), RemoteStorePathType.HASHED_PREFIX)
-            )
+            .setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX))
             .get();
 
         restoreSnapshotResponse = client.admin()
@@ -261,24 +260,25 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
             .get();
         assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
         ensureGreen(restoredIndexName1version2);
-        validateRemoteStorePathType(restoredIndexName1version2, RemoteStorePathType.HASHED_PREFIX);
+        validatePathType(restoredIndexName1version2, PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A);
 
         // Create index with cluster setting cluster.remote_store.index.path.prefix.type as hashed_prefix.
         indexSettings = getIndexSettings(1, 0).build();
         createIndex(indexName2, indexSettings);
         ensureGreen(indexName2);
-        validateRemoteStorePathType(indexName2, RemoteStorePathType.HASHED_PREFIX);
+        validatePathType(indexName2, PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A);
 
         // Validating that custom data has not changed for indexes which were created before the cluster setting got updated
-        validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);
+        validatePathType(indexName1, PathType.FIXED, PathHashAlgorithm.FNV_1A);
     }
 
-    private void validateRemoteStorePathType(String index, RemoteStorePathType pathType) {
+    private void validatePathType(String index, PathType pathType, PathHashAlgorithm pathHashAlgorithm) {
         ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();
         // Validate that the remote_store custom data is present in index metadata for the created index.
         Map<String, String> remoteCustomData = state.metadata().index(index).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
         assertNotNull(remoteCustomData);
-        assertEquals(pathType.toString(), remoteCustomData.get(RemoteStorePathType.NAME));
+        assertEquals(pathType.name(), remoteCustomData.get(PathType.NAME));
+        assertEquals(pathHashAlgorithm.name(), remoteCustomData.get(PathHashAlgorithm.NAME));
     }
 
     public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {

diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -90,8 +90,10 @@
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.MapperService.MergeReason;
 import org.opensearch.index.query.QueryShardContext;
-import org.opensearch.index.remote.RemoteStorePathType;
-import org.opensearch.index.remote.RemoteStorePathTypeResolver;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
+import org.opensearch.index.remote.RemoteStorePathStrategyResolver;
 import org.opensearch.index.shard.IndexSettingProvider;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.IndexCreationException;
@@ -175,7 +177,7 @@ public class MetadataCreateIndexService {
     private AwarenessReplicaBalance awarenessReplicaBalance;
 
     @Nullable
-    private final RemoteStorePathTypeResolver remoteStorePathTypeResolver;
+    private final RemoteStorePathStrategyResolver remoteStorePathStrategyResolver;
 
     public MetadataCreateIndexService(
         final Settings settings,
@@ -208,8 +210,8 @@ public MetadataCreateIndexService(
 
         // Task is onboarded for throttling, it will get retried from associated TransportClusterManagerNodeAction.
         createIndexTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.CREATE_INDEX_KEY, true);
-        remoteStorePathTypeResolver = isRemoteDataAttributePresent(settings)
-            ? new RemoteStorePathTypeResolver(clusterService.getClusterSettings())
+        remoteStorePathStrategyResolver = isRemoteDataAttributePresent(settings)
+            ? new RemoteStorePathStrategyResolver(clusterService.getClusterSettings())
             : null;
     }
 
@@ -558,7 +560,7 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
         tmpImdBuilder.setRoutingNumShards(routingNumShards);
         tmpImdBuilder.settings(indexSettings);
         tmpImdBuilder.system(isSystem);
-        addRemoteStorePathTypeInCustomData(tmpImdBuilder, true);
+        addRemoteStorePathStrategyInCustomData(tmpImdBuilder, true);
 
         // Set up everything, now locally create the index to see that things are ok, and apply
         IndexMetadata tempMetadata = tmpImdBuilder.build();
@@ -573,19 +575,27 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
      * @param tmpImdBuilder     index metadata builder.
      * @param assertNullOldType flag to verify that the old remote store path type is null
      */
-    public void addRemoteStorePathTypeInCustomData(IndexMetadata.Builder tmpImdBuilder, boolean assertNullOldType) {
-        if (remoteStorePathTypeResolver != null) {
+    public void addRemoteStorePathStrategyInCustomData(IndexMetadata.Builder tmpImdBuilder, boolean assertNullOldType) {
+        if (remoteStorePathStrategyResolver != null) {
             // It is possible that remote custom data exists already. In such cases, we need to only update the path type
             // in the remote store custom data map.
             Map<String, String> existingRemoteCustomData = tmpImdBuilder.removeCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
             Map<String, String> remoteCustomData = existingRemoteCustomData == null
                 ? new HashMap<>()
                 : new HashMap<>(existingRemoteCustomData);
             // Determine the path type for use using the remoteStorePathResolver.
-            String newPathType = remoteStorePathTypeResolver.getType().toString();
-            String oldPathType = remoteCustomData.put(RemoteStorePathType.NAME, newPathType);
-            assert !assertNullOldType || Objects.isNull(oldPathType);
-            logger.trace(() -> new ParameterizedMessage("Added new path type {}, replaced old path type {}", newPathType, oldPathType));
+            RemoteStorePathStrategy newPathStrategy = remoteStorePathStrategyResolver.get();
+            String oldPathType = remoteCustomData.put(PathType.NAME, newPathStrategy.getType().name());
+            String oldHashAlgorithm = remoteCustomData.put(PathHashAlgorithm.NAME, newPathStrategy.getHashAlgorithm().name());
+            assert !assertNullOldType || (Objects.isNull(oldPathType) && Objects.isNull(oldHashAlgorithm));
+            logger.trace(
+                () -> new ParameterizedMessage(
+                    "Added newPathStrategy={}, replaced oldPathType={} oldHashAlgorithm={}",
+                    newPathStrategy,
+                    oldPathType,
+                    oldHashAlgorithm
+                )
+            );
             tmpImdBuilder.putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, remoteCustomData);
         }
     }

diff --git a/server/src/main/java/org/opensearch/common/hash/FNV1a.java b/server/src/main/java/org/opensearch/common/hash/FNV1a.java
@@ -0,0 +1,48 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.hash;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Provides hashing function using FNV1a hash function. @see <a href="http://www.isthe.com/chongo/tech/comp/fnv/#FNV-1">FNV author's website</a>.
+ * 32 bit Java port of http://www.isthe.com/chongo/src/fnv/hash_32a.c
+ * 64 bit Java port of http://www.isthe.com/chongo/src/fnv/hash_64a.c
+ *
+ * @opensearch.internal
+ */
+public class FNV1a {
+    private static final long FNV_OFFSET_BASIS_32 = 0x811c9dc5L;
+    private static final long FNV_PRIME_32 = 0x01000193L;
+
+    private static final long FNV_OFFSET_BASIS_64 = 0xcbf29ce484222325L;
+    private static final long FNV_PRIME_64 = 0x100000001b3L;
+
+    // FNV-1a hash computation for 32-bit hash
+    public static long hash32(String input) {
+        long hash = FNV_OFFSET_BASIS_32;
+        byte[] bytes = input.getBytes(StandardCharsets.UTF_8);
+        for (byte b : bytes) {
+            hash ^= (b & 0xFF);
+            hash *= FNV_PRIME_32;
+        }
+        return hash;
+    }
+
+    // FNV-1a hash computation for 64-bit hash
+    public static long hash64(String input) {
+        long hash = FNV_OFFSET_BASIS_64;
+        byte[] bytes = input.getBytes(StandardCharsets.UTF_8);
+        for (byte b : bytes) {
+            hash ^= (b & 0xFF);
+            hash *= FNV_PRIME_64;
+        }
+        return hash;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java
@@ -511,7 +511,7 @@ public synchronized IndexShard createShard(
                         RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(this.indexSettings.getNodeSettings()),
                         this.indexSettings.getUUID(),
                         shardId,
-                        this.indexSettings.getRemoteStorePathType()
+                        this.indexSettings.getRemoteStorePathStrategy()
                     );
                 }
                 remoteStore = new Store(shardId, this.indexSettings, remoteDirectory, lock, Store.OnClose.EMPTY, path);

diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -49,7 +49,9 @@
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.index.Index;
-import org.opensearch.index.remote.RemoteStorePathType;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.ingest.IngestService;
@@ -1906,10 +1908,15 @@ public void setDocIdFuzzySetFalsePositiveProbability(double docIdFuzzySetFalsePo
         this.docIdFuzzySetFalsePositiveProbability = docIdFuzzySetFalsePositiveProbability;
     }
 
-    public RemoteStorePathType getRemoteStorePathType() {
+    public RemoteStorePathStrategy getRemoteStorePathStrategy() {
         Map<String, String> remoteCustomData = indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
-        return remoteCustomData != null && remoteCustomData.containsKey(RemoteStorePathType.NAME)
-            ? RemoteStorePathType.parseString(remoteCustomData.get(RemoteStorePathType.NAME))
-            : RemoteStorePathType.FIXED;
+        if (remoteCustomData != null
+            && remoteCustomData.containsKey(PathType.NAME)
+            && remoteCustomData.containsKey(PathHashAlgorithm.NAME)) {
+            PathType pathType = PathType.parseString(remoteCustomData.get(PathType.NAME));
+            PathHashAlgorithm pathHashAlgorithm = PathHashAlgorithm.parseString(remoteCustomData.get(PathHashAlgorithm.NAME));
+            return new RemoteStorePathStrategy(pathType, pathHashAlgorithm);
+        }
+        return new RemoteStorePathStrategy(PathType.FIXED);
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreDataEnums.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreDataEnums.java