From 678c225e8422ccb0388b8d51ec3c721396ec5c7b Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Tue, 15 Oct 2024 16:06:51 +0530 Subject: [PATCH] Do not delete last metadata file when pinned timestamps is enabled Signed-off-by: Sachin Kale --- .../opensearch/remotestore/RestoreShallowSnapshotV2IT.java | 4 ++-- .../opensearch/index/store/RemoteSegmentStoreDirectory.java | 5 +++++ .../index/translog/RemoteFsTimestampAwareTranslog.java | 2 +- .../index/translog/RemoteFsTimestampAwareTranslogTests.java | 4 ++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java index 460d7947ee427..d532abaa2b0ad 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java @@ -946,9 +946,9 @@ public void testContinuousIndexing() throws Exception { logger.info("Snapshots Status: " + snapshots); - for (String snapshot: snapshots.keySet()) { + for (String snapshot : snapshots.keySet()) { logger.info("Restoring snapshot: {}", snapshot); - assertAcked(client().admin().indices().prepareClose(index)); + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(index)).get()); RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin() .cluster() diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 6559f46bfc868..3c0fe69e9ab17 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -39,6 +39,7 @@ import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; +import org.opensearch.indices.RemoteStoreSettings; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService; import org.opensearch.threadpool.ThreadPool; @@ -896,6 +897,10 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException return; } + if (RemoteStoreSettings.isPinnedTimestampsEnabled()) { + metadataFilesEligibleToDelete.remove(sortedMetadataFileList.get(0)); + } + List metadataFilesToBeDeleted = metadataFilesEligibleToDelete.stream() .filter(metadataFile -> allLockFiles.contains(metadataFile) == false) .collect(Collectors.toList()); diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index 3ccacde22bbfc..ce08f81689b4f 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -189,7 +189,7 @@ public void onResponse(List blobMetadata) { List metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles, indexDeleted); // If index is not deleted, make sure to keep latest metadata file - if (indexDeleted == false) { + if (indexDeleted == false || RemoteStoreSettings.isPinnedTimestampsEnabled()) { metadataFilesToBeDeleted.remove(metadataFiles.get(0)); } diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index 73db3314f4d1e..01da1774f6800 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -285,9 +285,9 @@ public void testIndexDeletionWithNoPinnedTimestampNoRecentMdFiles() throws Excep assertBusy(() -> assertTrue(translog.isRemoteGenerationDeletionPermitsAvailable())); assertBusy(() -> { - assertEquals(0, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); + assertEquals(1, blobStoreTransferService.listAll(getTranslogDirectory().add(METADATA_DIR)).size()); assertEquals( - 0, + 12, blobStoreTransferService.listAll(getTranslogDirectory().add(DATA_DIR).add(String.valueOf(primaryTerm.get()))).size() ); });