From d5b4adfe73ff2b24e943bd607b848eee876d6db8 Mon Sep 17 00:00:00 2001 From: Finn Carroll Date: Tue, 6 Aug 2024 23:27:54 +0000 Subject: [PATCH] Test strict cache does not overflow Signed-off-by: Finn Carroll --- ...ansferManagerBlobContainerReaderTests.java | 5 ++-- ...sferManagerRemoteDirectoryReaderTests.java | 7 +++-- .../remote/utils/TransferManagerTestCase.java | 30 +++++++++++++++++-- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerBlobContainerReaderTests.java b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerBlobContainerReaderTests.java index 24e57cde3ffda..5386aba475934 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerBlobContainerReaderTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerBlobContainerReaderTests.java @@ -9,6 +9,7 @@ package org.opensearch.index.store.remote.utils; import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.index.store.remote.filecache.FileCache; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -24,10 +25,10 @@ public class TransferManagerBlobContainerReaderTests extends TransferManagerTest private BlobContainer blobContainer; @Override - protected void initializeTransferManager() throws IOException { + protected TransferManager initializeTransferManager(FileCache cache) throws IOException { blobContainer = mock(BlobContainer.class); doAnswer(i -> new ByteArrayInputStream(createData())).when(blobContainer).readBlob(eq("blob"), anyLong(), anyLong()); - transferManager = new TransferManager(blobContainer::readBlob, fileCache); + return new TransferManager(blobContainer::readBlob, cache); } protected void mockExceptionWhileReading() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerRemoteDirectoryReaderTests.java b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerRemoteDirectoryReaderTests.java index e777a287bf10f..4faf4fd9d0ddf 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerRemoteDirectoryReaderTests.java +++ b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerRemoteDirectoryReaderTests.java @@ -12,6 +12,7 @@ import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.common.lucene.store.InputStreamIndexInput; import org.opensearch.index.store.RemoteDirectory; +import org.opensearch.index.store.remote.filecache.FileCache; import java.io.IOException; import java.util.concurrent.CountDownLatch; @@ -26,15 +27,15 @@ public class TransferManagerRemoteDirectoryReaderTests extends TransferManagerTe private RemoteDirectory remoteDirectory; @Override - protected void initializeTransferManager() throws IOException { + protected TransferManager initializeTransferManager(FileCache cache) throws IOException { remoteDirectory = mock(RemoteDirectory.class); doAnswer(i -> new ByteArrayIndexInput("blob", createData())).when(remoteDirectory).openInput(eq("blob"), any()); - transferManager = new TransferManager( + return new TransferManager( (name, position, length) -> new InputStreamIndexInput( remoteDirectory.openInput(name, new BlockIOContext(IOContext.DEFAULT, position, length)), length ), - fileCache + cache ); } diff --git a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTestCase.java b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTestCase.java index 810a4c336fdf7..612041aa1e781 100644 --- a/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTestCase.java +++ b/server/src/test/java/org/opensearch/index/store/remote/utils/TransferManagerTestCase.java @@ -49,8 +49,9 @@ public abstract class TransferManagerTestCase extends OpenSearchTestCase { @Before public void setUp() throws Exception { super.setUp(); + fileCache.enableOverflow(true); directory = new MMapDirectory(createTempDir(), SimpleFSLockFactory.INSTANCE); - initializeTransferManager(); + transferManager = initializeTransferManager(fileCache); } @After @@ -64,6 +65,25 @@ protected static byte[] createData() { return data; } + public void testOverflowDisabled() throws Exception { + FileCache strictCache = FileCacheFactory.createConcurrentLRUFileCache( + EIGHT_MB * 2, + 1, + new NoopCircuitBreaker(CircuitBreaker.REQUEST) + ); + strictCache.enableOverflow(false); + TransferManager tm = initializeTransferManager(strictCache); + + assertThrows(IOException.class, () -> { + IndexInput i1 = fetchBlobWithName(tm, "1"); + IndexInput i2 = fetchBlobWithName(tm, "2"); + IndexInput i3 = fetchBlobWithName(tm, "3"); + assertIndexInputIsFunctional(i1); + assertIndexInputIsFunctional(i2); + assertIndexInputIsFunctional(i3); + }); + } + public void testSingleAccess() throws Exception { try (IndexInput i = fetchBlobWithName("file")) { assertIndexInputIsFunctional(i); @@ -193,7 +213,7 @@ public void testFetchesToDifferentBlobsDoNotBlockOnEachOther() throws Exception assertFalse(blockingThread.isAlive()); } - protected abstract void initializeTransferManager() throws IOException; + protected abstract TransferManager initializeTransferManager(FileCache cache) throws IOException; protected abstract void mockExceptionWhileReading() throws IOException; @@ -205,6 +225,12 @@ private IndexInput fetchBlobWithName(String blobname) throws IOException { return transferManager.fetchBlob(BlobFetchRequest.builder().fileName(blobname).directory(directory).blobParts(blobParts).build()); } + private IndexInput fetchBlobWithName(TransferManager tm, String blobname) throws IOException { + List blobParts = new ArrayList<>(); + blobParts.add(new BlobFetchRequest.BlobPart("blob", 0, EIGHT_MB)); + return tm.fetchBlob(BlobFetchRequest.builder().fileName(blobname).directory(directory).blobParts(blobParts).build()); + } + private static void assertIndexInputIsFunctional(IndexInput indexInput) throws IOException { indexInput.seek(EIGHT_MB - 1); MatcherAssert.assertThat(indexInput.readByte(), equalTo((byte) 7));