verify chunk size

Signed-off-by: panguixin <[email protected]>
opensearch-project · Jan 4, 2024 · 6f7822f · 6f7822f
1 parent 16d457d
commit 6f7822f
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 3 deletions.
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java
@@ -7,6 +7,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 
+import org.opensearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
 import org.opensearch.action.admin.cluster.node.stats.NodeStats;
 import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
@@ -16,6 +17,7 @@
 import org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
 import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
 import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
@@ -35,6 +37,7 @@
 import org.opensearch.index.IndexModule;
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter;
+import org.opensearch.index.store.remote.file.OnDemandBlockSnapshotIndexInput;
 import org.opensearch.index.store.remote.filecache.FileCacheStats;
 import org.opensearch.monitor.fs.FsInfo;
 import org.opensearch.node.Node;
@@ -75,10 +78,10 @@ protected Settings.Builder randomRepositorySettings() {
         return settings;
     }
 
-    private Settings.Builder chunkedRepositorySettings() {
+    private Settings.Builder chunkedRepositorySettings(long chunkSize) {
         final Settings.Builder settings = Settings.builder();
         settings.put("location", randomRepoPath()).put("compress", randomBoolean());
-        settings.put("chunk_size", 2 << 23, ByteSizeUnit.BYTES);
+        settings.put("chunk_size", chunkSize, ByteSizeUnit.BYTES);
         return settings;
     }
 
@@ -195,7 +198,8 @@ public void testCreateSearchableSnapshotWithChunks() throws Exception {
         final String snapshotName = "test-snap";
         final Client client = client();
 
-        Settings.Builder repositorySettings = chunkedRepositorySettings();
+        final int blockSize = OnDemandBlockSnapshotIndexInput.Builder.DEFAULT_BLOCK_SIZE;
+        Settings.Builder repositorySettings = chunkedRepositorySettings(blockSize * randomLongBetween(1, 100));
 
         internalCluster().ensureAtLeastNumSearchAndDataNodes(numReplicasIndex + 1);
         createIndexWithDocsAndEnsureGreen(numReplicasIndex, 1000, indexName);
@@ -209,6 +213,39 @@ public void testCreateSearchableSnapshotWithChunks() throws Exception {
         assertDocCount(restoredIndexName, 1000L);
     }
 
+    /**
+     * Tests a chunked repository scenario where the chunk size of repo is not a multiple of the block size.
+     */
+    public void testCreateSearchableSnapshotWithChunksThrowExceptions() throws Exception {
+        final int numReplicasIndex = randomIntBetween(1, 4);
+        final String indexName = "test-idx";
+        final String restoredIndexName = indexName + "-copy";
+        final String repoName = "test-repo";
+        final String snapshotName = "test-snap";
+        final Client client = client();
+
+        final int blockSize = OnDemandBlockSnapshotIndexInput.Builder.DEFAULT_BLOCK_SIZE;
+        Settings.Builder repositorySettings = chunkedRepositorySettings(blockSize + 1);
+
+        internalCluster().ensureAtLeastNumSearchAndDataNodes(numReplicasIndex + 1);
+        createIndexWithDocsAndEnsureGreen(numReplicasIndex, 1000, indexName);
+        createRepositoryWithSettings(repositorySettings, repoName);
+        takeSnapshot(client, snapshotName, repoName, indexName);
+
+        deleteIndicesAndEnsureGreen(client, indexName);
+        RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshot(client, snapshotName, repoName);
+        assertEquals(0, restoreSnapshotResponse.getRestoreInfo().successfulShards());
+
+        ClusterAllocationExplainResponse explainResponse = client.admin().cluster().prepareAllocationExplain().execute().actionGet();
+        String detailsMessage = explainResponse.getExplanation().getUnassignedInfo().getDetails();
+        assert detailsMessage != null;
+        assertTrue(
+            detailsMessage.contains(
+                "the chunk size of snapshot used for searchable snapshot index must be either the Long.MAX_VALUE or a multiple of the OnDemandBlockIndexInput block size"
+            )
+        );
+    }
+
     /**
      * Tests the functionality of remote shard allocation to
      * ensure it can assign remote shards to a node with local shards given it has the
@@ -399,6 +436,19 @@ private void restoreSnapshotAndEnsureGreen(Client client, String snapshotName, S
         ensureGreen();
     }
 
+    private RestoreSnapshotResponse restoreSnapshot(Client client, String snapshotName, String repoName) {
+        logger.info("--> restore indices as 'remote_snapshot'");
+        return client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(repoName, snapshotName)
+            .setRenamePattern("(.+)")
+            .setRenameReplacement("$1-copy")
+            .setStorageType(RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT)
+            .setWaitForCompletion(true)
+            .execute()
+            .actionGet();
+    }
+
     private void assertRemoteSnapshotIndexSettings(Client client, String... snapshotIndexNames) {
         GetSettingsResponse settingsResponse = client.admin()
             .indices()

diff --git a/...er/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectory.java b/...er/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectory.java
@@ -47,6 +47,18 @@ public final class RemoteSnapshotDirectory extends Directory {
     private final TransferManager transferManager;
 
     public RemoteSnapshotDirectory(BlobStoreIndexShardSnapshot snapshot, FSDirectory localStoreDir, TransferManager transferManager) {
+        final long chunkSize = snapshot.indexFiles().isEmpty() ? 0L : snapshot.indexFiles().get(0).partSize().getBytes();
+        final long blockSize = OnDemandBlockSnapshotIndexInput.Builder.DEFAULT_BLOCK_SIZE;
+        if ((chunkSize & (blockSize - 1)) != 0 && chunkSize != Long.MAX_VALUE) {
+            throw new IllegalArgumentException(
+                "the chunk size of snapshot used for searchable snapshot index must be either the Long.MAX_VALUE or a multiple of the OnDemandBlockIndexInput block size, got chunk size ["
+                    + chunkSize
+                    + "], and block size ["
+                    + blockSize
+                    + "]"
+            );
+        }
+
         this.fileInfoMap = snapshot.indexFiles()
             .stream()
             .collect(Collectors.toMap(BlobStoreIndexShardSnapshot.FileInfo::physicalName, f -> f));