Skip to content

Commit

Permalink
Treat last fetch timestamp of pinned timestamp as one of the pinned t…
Browse files Browse the repository at this point in the history 
…imestamps (#16392)

Signed-off-by: Sachin Kale <[email protected]>
(cherry picked from commit 78d2a4e)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
  • Loading branch information
@github-actions
github-actions[bot] committed Oct 21, 2024
1 parent 3c5e054 commit f4c6be7
Show file tree
Hide file tree
Showing 4 changed files with 165 additions and 42 deletions.
110 changes: 109 additions & 1 deletion ...r/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -927,7 +927,7 @@ public void testContinuousIndexing() throws Exception {
int numDocs = randomIntBetween(200, 300);
totalDocs += numDocs;
try (BackgroundIndexer indexer = new BackgroundIndexer(index, MapperService.SINGLE_MAPPING_NAME, client(), numDocs)) {
int numberOfSnapshots = 5;
int numberOfSnapshots = 2;
for (int i = 0; i < numberOfSnapshots; i++) {
logger.info("--> waiting for {} docs to be indexed ...", numDocs);
long finalTotalDocs1 = totalDocs;
Expand Down Expand Up @@ -976,4 +976,112 @@ public void testContinuousIndexing() throws Exception {
});
}
}

public void testHashedPrefixTranslogMetadataCombination() throws Exception {
Settings settings = Settings.builder()
.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(RemoteStoreEnums.PathType.values()))
.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), randomBoolean())
.build();

internalCluster().startClusterManagerOnlyNode(settings);
internalCluster().startDataOnlyNode(settings);
String index = "test-index";
String snapshotRepo = "test-restore-snapshot-repo";
String baseSnapshotName = "snapshot_";
Path absolutePath1 = randomRepoPath().toAbsolutePath();
logger.info("Snapshot Path [{}]", absolutePath1);

createRepository(snapshotRepo, "fs", getRepositorySettings(absolutePath1, true));

Client client = client();
Settings indexSettings = Settings.builder()
.put(super.indexSettings())
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.build();

createIndex(index, indexSettings);
ensureGreen(index);

RemoteStorePinnedTimestampService remoteStorePinnedTimestampService = internalCluster().getInstance(
RemoteStorePinnedTimestampService.class,
primaryNodeName(index)
);

remoteStorePinnedTimestampService.rescheduleAsyncUpdatePinnedTimestampTask(TimeValue.timeValueSeconds(randomIntBetween(1, 5)));
RemoteStoreSettings.setPinnedTimestampsLookbackInterval(TimeValue.timeValueSeconds(randomIntBetween(1, 5)));

long totalDocs = 0;
Map<String, Long> snapshots = new HashMap<>();
int numDocs = randomIntBetween(200, 300);
totalDocs += numDocs;
try (BackgroundIndexer indexer = new BackgroundIndexer(index, MapperService.SINGLE_MAPPING_NAME, client(), numDocs)) {
int numberOfSnapshots = 2;
for (int i = 0; i < numberOfSnapshots; i++) {
logger.info("--> waiting for {} docs to be indexed ...", numDocs);
long finalTotalDocs1 = totalDocs;
assertBusy(() -> assertEquals(finalTotalDocs1, indexer.totalIndexedDocs()), 120, TimeUnit.SECONDS);
logger.info("--> {} total docs indexed", totalDocs);
String snapshotName = baseSnapshotName + i;
createSnapshot(snapshotRepo, snapshotName, new ArrayList<>());
snapshots.put(snapshotName, totalDocs);
if (i < numberOfSnapshots - 1) {
numDocs = randomIntBetween(200, 300);
indexer.continueIndexing(numDocs);
totalDocs += numDocs;
}
}
}

logger.info("Snapshots Status: " + snapshots);

for (String snapshot : snapshots.keySet()) {
logger.info("Restoring snapshot: {}", snapshot);

if (randomBoolean()) {
assertAcked(client().admin().indices().delete(new DeleteIndexRequest(index)).get());
} else {
assertAcked(client().admin().indices().prepareClose(index));
}

assertTrue(
internalCluster().client()
.admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(RemoteStoreEnums.PathType.values()))
.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), randomBoolean())
)
.get()
.isAcknowledged()
);

RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepo, snapshot)
.setWaitForCompletion(true)
.setIndices()
.get();

assertEquals(RestStatus.OK, restoreSnapshotResponse1.status());

// Verify restored index's stats
ensureGreen(TimeValue.timeValueSeconds(60), index);
long finalTotalDocs = totalDocs;
assertBusy(() -> {
Long hits = client().prepareSearch(index)
.setQuery(matchAllQuery())
.setSize((int) finalTotalDocs)
.storedFields()
.execute()
.actionGet()
.getHits()
.getTotalHits().value;

assertEquals(snapshots.get(snapshot), hits);
});
}
}
}
10 changes: 3 additions & 7 deletions server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
import org.opensearch.index.store.lockmanager.RemoteStoreMetadataLockManager;
import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata;
import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler;
import org.opensearch.indices.RemoteStoreSettings;
import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService;
import org.opensearch.threadpool.ThreadPool;
Expand Down Expand Up @@ -877,9 +876,11 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException

Tuple<Long, Set<Long>> pinnedTimestampsState = RemoteStorePinnedTimestampService.getPinnedTimestamps();

Set<Long> pinnedTimestamps = new HashSet<>(pinnedTimestampsState.v2());
pinnedTimestamps.add(pinnedTimestampsState.v1());
Set<String> implicitLockedFiles = RemoteStoreUtils.getPinnedTimestampLockedFiles(
sortedMetadataFileList,
pinnedTimestampsState.v2(),
pinnedTimestamps,
metadataFilePinnedTimestampMap,
MetadataFilenameUtils::getTimestamp,
MetadataFilenameUtils::getNodeIdByPrimaryTermAndGen
Expand Down Expand Up @@ -912,11 +913,6 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException
return;
}

// If pinned timestamps are enabled, make sure to not delete last metadata file.
if (RemoteStoreSettings.isPinnedTimestampsEnabled()) {
metadataFilesEligibleToDelete.remove(sortedMetadataFileList.get(0));
}

List<String> metadataFilesToBeDeleted = metadataFilesEligibleToDelete.stream()
.filter(metadataFile -> allLockFiles.contains(metadataFile) == false)
.collect(Collectors.toList());
Expand Down
6 changes: 4 additions & 2 deletions server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ public void onResponse(List<BlobMetadata> blobMetadata) {
List<String> metadataFilesToBeDeleted = getMetadataFilesToBeDeleted(metadataFiles, indexDeleted);

// If index is not deleted, make sure to keep latest metadata file
if (indexDeleted == false || RemoteStoreSettings.isPinnedTimestampsEnabled()) {
if (indexDeleted == false) {
metadataFilesToBeDeleted.remove(metadataFiles.get(0));
}

Expand Down Expand Up @@ -345,9 +345,11 @@ protected static List<String> getMetadataFilesToBeDeleted(
);

// Get md files matching pinned timestamps
Set<Long> pinnedTimestamps = new HashSet<>(pinnedTimestampsState.v2());
pinnedTimestamps.add(pinnedTimestampsState.v1());
Set<String> implicitLockedFiles = RemoteStoreUtils.getPinnedTimestampLockedFiles(
metadataFilesToBeDeleted,
pinnedTimestampsState.v2(),
pinnedTimestamps,
metadataFilePinnedTimestampMap,
file -> RemoteStoreUtils.invertLong(file.split(METADATA_SEPARATOR)[3]),
TranslogTransferMetadata::getNodeIdByPrimaryTermAndGen
Expand Down
81 changes: 49 additions & 32 deletions server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -755,17 +755,21 @@ public void testGetGenerationsToBeDeletedWithGenerationInRemote() throws IOExcep
assertTrue(generations.isEmpty());
}

public void testGetMetadataFilesToBeDeletedNoExclusion() {
public void testGetMetadataFilesToBeDeletedExclusionDueToRefreshTimestamp() {
updatePinnedTimstampTask.run();

List<String> metadataFiles = List.of(
"metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__9223372036854775106__1",
"metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__9223372036854775803__1",
"metadata__9223372036438563903__9223372036854775701__9223370311919910403__31__9223372036854775701__1"
);
List<String> metadataFiles = new ArrayList<>();
metadataFiles.add("metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__9223372036854775106__1");
metadataFiles.add("metadata__9223372036438563903__9223372036854775701__9223370311919910403__31__9223372036854775701__1");
metadataFiles.add("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__9223372036854775803__1");

// Removing file that is pinned by latest refresh timestamp
List<String> metadataFilesToBeDeleted = new ArrayList<>(metadataFiles);
metadataFilesToBeDeleted.remove(
"metadata__9223372036438563903__9223372036854774799__9223370311919910393__31__9223372036854775106__1"
);
assertEquals(
metadataFiles,
metadataFilesToBeDeleted,
RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(metadataFiles, new HashMap<>(), Long.MAX_VALUE, false, logger)
);
}
Expand All @@ -774,13 +778,15 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() {
updatePinnedTimstampTask.run();
long currentTimeInMillis = System.currentTimeMillis();
String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000);
String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 30000);
String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 60000);
String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 400000);
String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 30000);
String md4Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis + 60000);

List<String> metadataFiles = List.of(
"metadata__9223372036438563903__9223372036854774799__" + md1Timestamp + "__31__9223372036854775106__1",
"metadata__9223372036438563903__9223372036854775800__" + md2Timestamp + "__31__9223372036854775803__1",
"metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1"
"metadata__9223372036438563903__9223372036854774500__" + md1Timestamp + "__31__9223372036854775106__1",
"metadata__9223372036438563903__9223372036854774799__" + md2Timestamp + "__31__9223372036854775106__1",
"metadata__9223372036438563903__9223372036854775800__" + md3Timestamp + "__31__9223372036854775803__1",
"metadata__9223372036438563903__9223372036854775701__" + md4Timestamp + "__31__9223372036854775701__1"
);

List<String> metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(
Expand All @@ -791,24 +797,26 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeOnly() {
logger
);
assertEquals(1, metadataFilesToBeDeleted.size());
assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0));
assertEquals(metadataFiles.get(1), metadataFilesToBeDeleted.get(0));
}

public void testGetMetadataFilesToBeDeletedExclusionBasedOnPinningOnly() throws IOException {
long currentTimeInMillis = System.currentTimeMillis();
String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000);
String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000);
String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000);
String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 190000);
String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000);
String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000);
String md4Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000);

long pinnedTimestamp = RemoteStoreUtils.invertLong(md2Timestamp) + 10000;
long pinnedTimestamp = RemoteStoreUtils.invertLong(md3Timestamp) + 10000;
when(blobContainer.listBlobs()).thenReturn(Map.of(randomInt(100) + "__" + pinnedTimestamp, new PlainBlobMetadata("xyz", 100)));

updatePinnedTimstampTask.run();

List<String> metadataFiles = List.of(
"metadata__9223372036438563903__9223372036854774799__" + md1Timestamp + "__31__9223372036854775106__1",
"metadata__9223372036438563903__9223372036854775600__" + md2Timestamp + "__31__9223372036854775803__1",
"metadata__9223372036438563903__9223372036854775701__" + md3Timestamp + "__31__9223372036854775701__1"
"metadata__9223372036438563903__9223372036854774500__" + md1Timestamp + "__31__9223372036854775701__1",
"metadata__9223372036438563903__9223372036854774799__" + md2Timestamp + "__31__9223372036854775106__1",
"metadata__9223372036438563903__9223372036854775600__" + md3Timestamp + "__31__9223372036854775803__1",
"metadata__9223372036438563903__9223372036854775701__" + md4Timestamp + "__31__9223372036854775701__1"
);

List<String> metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(
Expand All @@ -819,8 +827,8 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnPinningOnly() throws
logger
);
assertEquals(2, metadataFilesToBeDeleted.size());
assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0));
assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(1));
assertEquals(metadataFiles.get(1), metadataFilesToBeDeleted.get(0));
assertEquals(metadataFiles.get(3), metadataFilesToBeDeleted.get(1));
}

public void testGetMetadataFilesToBeDeletedExclusionBasedOnAgeAndPinning() throws IOException {
Expand Down Expand Up @@ -856,6 +864,7 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationOnly() thro
String md1Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 200000);
String md2Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 300000);
String md3Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 600000);
String md4Timestamp = RemoteStoreUtils.invertLong(currentTimeInMillis - 800000);

when(blobContainer.listBlobs()).thenReturn(Map.of());

Expand All @@ -866,8 +875,10 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationOnly() thro
"metadata__9223372036438563903__9223372036854775800__" + md1Timestamp + "__31__9223372036854775106__1",
// MaxGen 12
"metadata__9223372036438563903__9223372036854775795__" + md2Timestamp + "__31__9223372036854775803__1",
// MaxGen 9
"metadata__9223372036438563903__9223372036854775798__" + md3Timestamp + "__31__9223372036854775701__1",
// MaxGen 10
"metadata__9223372036438563903__9223372036854775798__" + md3Timestamp + "__31__9223372036854775701__1"
"metadata__9223372036438563903__9223372036854775797__" + md4Timestamp + "__31__9223372036854775701__1"
);

List<String> metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(
Expand All @@ -878,8 +889,8 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationOnly() thro
logger
);
assertEquals(2, metadataFilesToBeDeleted.size());
assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(0));
assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(1));
assertEquals(metadataFiles.get(2), metadataFilesToBeDeleted.get(0));
assertEquals(metadataFiles.get(0), metadataFilesToBeDeleted.get(1));
}

public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationDeleteIndex() throws IOException {
Expand All @@ -892,13 +903,15 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationDeleteIndex

updatePinnedTimstampTask.run();

List<String> metadataFiles = List.of(
// MaxGen 7
"metadata__9223372036438563903__9223372036854775800__" + md1Timestamp + "__31__9223372036854775106__1",
// MaxGen 12
"metadata__9223372036438563903__9223372036854775795__" + md2Timestamp + "__31__9223372036854775803__1",
// MaxGen 17
"metadata__9223372036438563903__9223372036854775790__" + md3Timestamp + "__31__9223372036854775701__1"
List<String> metadataFiles = new ArrayList<>(
List.of(
// MaxGen 12
"metadata__9223372036438563903__9223372036854775795__" + md2Timestamp + "__31__9223372036854775803__1",
// MaxGen 7
"metadata__9223372036438563903__9223372036854775800__" + md1Timestamp + "__31__9223372036854775106__1",
// MaxGen 17
"metadata__9223372036438563903__9223372036854775790__" + md3Timestamp + "__31__9223372036854775701__1"
)
);

List<String> metadataFilesToBeDeleted = RemoteFsTimestampAwareTranslog.getMetadataFilesToBeDeleted(
Expand All @@ -908,6 +921,10 @@ public void testGetMetadataFilesToBeDeletedExclusionBasedOnGenerationDeleteIndex
true,
logger
);

// Metadata file corresponding to latest pinned timestamp fetch is always considered pinned
metadataFiles.remove(metadataFiles.get(2));

assertEquals(metadataFiles, metadataFilesToBeDeleted);
}

Expand Down

0 comments on commit f4c6be7

Please sign in to comment.