Skip to content

Commit

Permalink
Add unit tests for RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTer…
Browse files Browse the repository at this point in the history 
…mFromMetadataFile

Signed-off-by: Sachin Kale <[email protected]>
  • Loading branch information
@sachinpkale
sachinpkale committed Sep 30, 2024
1 parent d5deaf9 commit 2924dca
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 6 deletions.
13 changes: 7 additions & 6 deletions server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,7 @@ public void onResponse(List<BlobMetadata> blobMetadata) {

// Update cache to keep only those metadata files that are not getting deleted
oldFormatMetadataFileGenerationMap.keySet().retainAll(metadataFilesNotToBeDeleted);
oldFormatMetadataFilePrimaryTermMap.keySet().retainAll(metadataFilesNotToBeDeleted);
// Delete stale primary terms
deleteStaleRemotePrimaryTerms(metadataFilesNotToBeDeleted);
} else {
Expand Down Expand Up @@ -408,9 +409,9 @@ protected Tuple<Long, Long> getMinMaxTranslogGenerationFromMetadataFile(
}
}

private void deleteStaleRemotePrimaryTerms(List<String> metadataFiles) {
private void deleteStaleRemotePrimaryTerms(List<String> metadataFilesNotToBeDeleted) {
deleteStaleRemotePrimaryTerms(
metadataFiles,
metadataFilesNotToBeDeleted,
translogTransferManager,
oldFormatMetadataFilePrimaryTermMap,
minPrimaryTermInRemote,
Expand All @@ -425,7 +426,7 @@ private void deleteStaleRemotePrimaryTerms(List<String> metadataFiles) {
* This will also delete all stale translog metadata files from remote except the latest basis the metadata file comparator.
*/
protected static void deleteStaleRemotePrimaryTerms(
List<String> metadataFiles,
List<String> metadataFilesNotToBeDeleted,
TranslogTransferManager translogTransferManager,
Map<String, Tuple<Long, Long>> oldFormatMetadataFilePrimaryTermMap,
AtomicLong minPrimaryTermInRemoteAtomicLong,
Expand All @@ -434,15 +435,15 @@ protected static void deleteStaleRemotePrimaryTerms(
// The deletion of older translog files in remote store is on best-effort basis, there is a possibility that there
// are older files that are no longer needed and should be cleaned up. In here, we delete all files that are part
// of older primary term.
if (metadataFiles.isEmpty()) {
if (metadataFilesNotToBeDeleted.isEmpty()) {
logger.trace("No metadata is uploaded yet, returning from deleteStaleRemotePrimaryTerms");
return;
}
Optional<Long> minPrimaryTermFromMetadataFiles = metadataFiles.stream().map(file -> {
Optional<Long> minPrimaryTermFromMetadataFiles = metadataFilesNotToBeDeleted.stream().map(file -> {
try {
return getMinMaxPrimaryTermFromMetadataFile(file, translogTransferManager, oldFormatMetadataFilePrimaryTermMap).v1();
} catch (IOException e) {
return Long.MAX_VALUE;
return Long.MIN_VALUE;
}
}).min(Long::compareTo);
// First we delete all stale primary terms folders from remote store
Expand Down
59 changes: 59 additions & 0 deletions server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -999,6 +999,65 @@ public void testGetMinMaxTranslogGenerationFromMetadataFile() throws IOException
verify(translogTransferManager).readMetadata("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1");
}

public void testGetMinMaxPrimaryTermFromMetadataFile() throws IOException {
TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class);

RemoteFsTimestampAwareTranslog translog = (RemoteFsTimestampAwareTranslog) this.translog;

// Fetch generations directly from the filename
assertEquals(
new Tuple<>(1L, 1008L),
RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile(
"metadata__9223372036854774799__9223372036854774799__9223370311919910393__31__9223372036854775106__1__1",
translogTransferManager,
new HashMap<>()
)
);
assertEquals(
new Tuple<>(4L, 7L),
RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile(
"metadata__9223372036854775800__9223372036854775800__9223370311919910398__31__9223372036854775803__4__1",
translogTransferManager,
new HashMap<>()
)
);
assertEquals(
new Tuple<>(10L, 10L),
RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile(
"metadata__9223372036854775797__9223372036854775800__9223370311919910398__31__9223372036854775803__10__1",
translogTransferManager,
new HashMap<>()
)
);

// For older md filenames, it needs to read the content
TranslogTransferMetadata md1 = mock(TranslogTransferMetadata.class);
when(md1.getGenerationToPrimaryTermMapper()).thenReturn(Map.of("12", "1", "23", "1", "34", "2"));
when(translogTransferManager.readMetadata("metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1"))
.thenReturn(md1);
assertEquals(
new Tuple<>(1L, 2L),
RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile(
"metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1",
translogTransferManager,
new HashMap<>()
)
);
assertEquals(
new Tuple<>(4L, 7L),
RemoteFsTimestampAwareTranslog.getMinMaxPrimaryTermFromMetadataFile(
"metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1",
translogTransferManager,
Map.of("metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1", new Tuple<>(4L, 7L))
)
);

verify(translogTransferManager).readMetadata("metadata__9223372036854775805__9223372036854774799__9223370311919910393__31__1");
verify(translogTransferManager, times(0)).readMetadata(
"metadata__9223372036438563903__9223372036854775800__9223370311919910398__31__1"
);
}

public void testDeleteStaleRemotePrimaryTerms() throws IOException {
TranslogTransferManager translogTransferManager = mock(TranslogTransferManager.class);

Expand Down

0 comments on commit 2924dca

Please sign in to comment.