From 1e4927bcf4289cd6d0681bf74ce8f2625fd20550 Mon Sep 17 00:00:00 2001 From: Kiran Prakash Date: Thu, 28 Mar 2024 18:30:36 -0700 Subject: [PATCH] ITs Signed-off-by: Kiran Prakash --- .../indices/IndicesRequestCacheIT.java | 98 +++++++++++++++++-- 1 file changed, 91 insertions(+), 7 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java index 52b4dad553180..8c157f85defaa 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java @@ -34,6 +34,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.opensearch.action.search.SearchResponse; @@ -42,6 +43,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.cache.request.RequestCacheStats; import org.opensearch.index.query.QueryBuilders; @@ -49,9 +51,12 @@ import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.histogram.Histogram; import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket; +import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import org.opensearch.test.hamcrest.OpenSearchAssertions; +import java.time.Duration; +import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -60,6 +65,7 @@ import java.util.Collection; import java.util.List; +import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING; import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.dateRange; @@ -69,6 +75,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, supportsDedicatedMasters = false) public class IndicesRequestCacheIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase { public IndicesRequestCacheIT(Settings settings) { super(settings); @@ -676,15 +683,89 @@ public void testCacheWithInvalidation() throws Exception { // Should expect miss as key has changed due to change in IndexReader.CacheKey (due to refresh) assertCacheState(client, "index", 1, 2); } + // when staleness threshold is high, it should NOT clean-up + public void testStaleKeysCleanup_ThresholdUpdates() throws Exception { + Instant start = Instant.now(); + long thresholdInMillis = 1_500; + String node = internalCluster().startNode( + Settings.builder() + .put(IndicesRequestCache.SETTING_INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING, 0.90) + .put(IndicesRequestCache.SETTING_INDICES_REQUEST_CACHE_CLEAN_INTERVAL_SETTING, TimeValue.timeValueMillis(thresholdInMillis)) + ); + String index = "index"; + Client client = client(node); + setupIndex(client, index); + + // create first cache entry + createCacheEntry(client, index, "hello"); + assertCacheState(client, index, 0, 1); + long expectedFirstCachedItemEntrySize = getRequestCacheStats(client, index).getMemorySizeInBytes(); + assertTrue(expectedFirstCachedItemEntrySize > 0); + + // create second cache entry + createCacheEntry(client, index, "there"); + assertCacheState(client, "index", 0, 2); + assertEquals(expectedFirstCachedItemEntrySize * 2, getRequestCacheStats(client, "index").getMemorySizeInBytes()); + + // force refresh so that it creates 2 stale keys in the cache for the cache cleaner to pick up. + flushAndRefresh("index"); + client().prepareIndex("index").setId("1").setSource("k", "good bye"); + ensureSearchable("index"); + + // create another entry + createCacheEntry(client, index, "hello1"); + assertCacheState(client, "index", 0, 3); + long cacheSizeBeforeCleanup = getRequestCacheStats(client, "index").getMemorySizeInBytes(); + assertTrue(cacheSizeBeforeCleanup > expectedFirstCachedItemEntrySize * 2); + assertEquals(cacheSizeBeforeCleanup, expectedFirstCachedItemEntrySize * 3, 2); + + Instant end = Instant.now(); + long elapsedTimeMillis = Duration.between(start, end).toMillis(); + // if this test is flaky, increase the sleep time. + long sleepTime = (thresholdInMillis - elapsedTimeMillis) + 2_000; + Thread.sleep(sleepTime); + + // cache cleaner should have skipped the cleanup + long cacheSizeAfterCleanup = getRequestCacheStats(client, "index").getMemorySizeInBytes(); + assertEquals(cacheSizeBeforeCleanup, cacheSizeAfterCleanup); + + // Set indices.requests.cache.cleanup.staleness_threshold to "10%" + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings(Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), 0.10)); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + + Thread.sleep(1_500); + cacheSizeAfterCleanup = getRequestCacheStats(client, "index").getMemorySizeInBytes(); + assertTrue(cacheSizeBeforeCleanup > cacheSizeAfterCleanup); + } + + private void setupIndex(Client client, String index) throws Exception { + assertAcked( + client.admin() + .indices() + .prepareCreate(index) + .setMapping("k", "type=keyword") + .setSettings( + Settings.builder() + .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ) + .get() + ); + indexRandom(true, client.prepareIndex(index).setSource("k", "hello")); + indexRandom(true, client.prepareIndex(index).setSource("k", "there")); + ensureSearchable(index); + } + + private void createCacheEntry(Client client, String index, String value) { + SearchResponse resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", value)).get(); + assertSearchResponse(resp); + OpenSearchAssertions.assertAllSuccessful(resp); + } private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) { - RequestCacheStats requestCacheStats = client.admin() - .indices() - .prepareStats(index) - .setRequestCache(true) - .get() - .getTotal() - .getRequestCache(); + RequestCacheStats requestCacheStats = getRequestCacheStats(client, index); // Check the hit count and miss count together so if they are not // correct we can see both values assertEquals( @@ -694,4 +775,7 @@ private static void assertCacheState(Client client, String index, long expectedH } + private static RequestCacheStats getRequestCacheStats(Client client, String index) { + return client.admin().indices().prepareStats(index).setRequestCache(true).get().getTotal().getRequestCache(); + } }