Skip to content

Commit

Permalink
ITs
Browse files Browse the repository at this point in the history 
Signed-off-by: Kiran Prakash <[email protected]>
  • Loading branch information
@kiranprakash154
kiranprakash154 committed Mar 29, 2024
1 parent 268797f commit 1e4927b
Showing 1 changed file with 91 additions and 7 deletions.
98 changes: 91 additions & 7 deletions server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;

import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
import org.opensearch.action.admin.indices.alias.Alias;
import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse;
import org.opensearch.action.search.SearchResponse;
Expand All @@ -42,16 +43,20 @@
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.time.DateFormatter;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.util.FeatureFlags;
import org.opensearch.index.cache.request.RequestCacheStats;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.opensearch.search.aggregations.bucket.histogram.Histogram;
import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
import org.opensearch.test.OpenSearchIntegTestCase;
import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
import org.opensearch.test.hamcrest.OpenSearchAssertions;

import java.time.Duration;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
Expand All @@ -60,6 +65,7 @@
import java.util.Collection;
import java.util.List;

import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING;
import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram;
import static org.opensearch.search.aggregations.AggregationBuilders.dateRange;
Expand All @@ -69,6 +75,7 @@
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;

@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, supportsDedicatedMasters = false)
public class IndicesRequestCacheIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase {
public IndicesRequestCacheIT(Settings settings) {
super(settings);
Expand Down Expand Up @@ -676,15 +683,89 @@ public void testCacheWithInvalidation() throws Exception {
// Should expect miss as key has changed due to change in IndexReader.CacheKey (due to refresh)
assertCacheState(client, "index", 1, 2);
}
// when staleness threshold is high, it should NOT clean-up
public void testStaleKeysCleanup_ThresholdUpdates() throws Exception {
Instant start = Instant.now();
long thresholdInMillis = 1_500;
String node = internalCluster().startNode(
Settings.builder()
.put(IndicesRequestCache.SETTING_INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING, 0.90)
.put(IndicesRequestCache.SETTING_INDICES_REQUEST_CACHE_CLEAN_INTERVAL_SETTING, TimeValue.timeValueMillis(thresholdInMillis))
);
String index = "index";
Client client = client(node);
setupIndex(client, index);

// create first cache entry
createCacheEntry(client, index, "hello");
assertCacheState(client, index, 0, 1);
long expectedFirstCachedItemEntrySize = getRequestCacheStats(client, index).getMemorySizeInBytes();
assertTrue(expectedFirstCachedItemEntrySize > 0);

// create second cache entry
createCacheEntry(client, index, "there");
assertCacheState(client, "index", 0, 2);
assertEquals(expectedFirstCachedItemEntrySize * 2, getRequestCacheStats(client, "index").getMemorySizeInBytes());

// force refresh so that it creates 2 stale keys in the cache for the cache cleaner to pick up.
flushAndRefresh("index");
client().prepareIndex("index").setId("1").setSource("k", "good bye");
ensureSearchable("index");

// create another entry
createCacheEntry(client, index, "hello1");
assertCacheState(client, "index", 0, 3);
long cacheSizeBeforeCleanup = getRequestCacheStats(client, "index").getMemorySizeInBytes();
assertTrue(cacheSizeBeforeCleanup > expectedFirstCachedItemEntrySize * 2);
assertEquals(cacheSizeBeforeCleanup, expectedFirstCachedItemEntrySize * 3, 2);

Instant end = Instant.now();
long elapsedTimeMillis = Duration.between(start, end).toMillis();
// if this test is flaky, increase the sleep time.
long sleepTime = (thresholdInMillis - elapsedTimeMillis) + 2_000;
Thread.sleep(sleepTime);

// cache cleaner should have skipped the cleanup
long cacheSizeAfterCleanup = getRequestCacheStats(client, "index").getMemorySizeInBytes();
assertEquals(cacheSizeBeforeCleanup, cacheSizeAfterCleanup);

// Set indices.requests.cache.cleanup.staleness_threshold to "10%"
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
updateSettingsRequest.persistentSettings(Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), 0.10));
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());

Thread.sleep(1_500);
cacheSizeAfterCleanup = getRequestCacheStats(client, "index").getMemorySizeInBytes();
assertTrue(cacheSizeBeforeCleanup > cacheSizeAfterCleanup);
}

private void setupIndex(Client client, String index) throws Exception {
assertAcked(
client.admin()
.indices()
.prepareCreate(index)
.setMapping("k", "type=keyword")
.setSettings(
Settings.builder()
.put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
)
.get()
);
indexRandom(true, client.prepareIndex(index).setSource("k", "hello"));
indexRandom(true, client.prepareIndex(index).setSource("k", "there"));
ensureSearchable(index);
}

private void createCacheEntry(Client client, String index, String value) {
SearchResponse resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", value)).get();
assertSearchResponse(resp);
OpenSearchAssertions.assertAllSuccessful(resp);
}

private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {
RequestCacheStats requestCacheStats = client.admin()
.indices()
.prepareStats(index)
.setRequestCache(true)
.get()
.getTotal()
.getRequestCache();
RequestCacheStats requestCacheStats = getRequestCacheStats(client, index);
// Check the hit count and miss count together so if they are not
// correct we can see both values
assertEquals(
Expand All @@ -694,4 +775,7 @@ private static void assertCacheState(Client client, String index, long expectedH

}

private static RequestCacheStats getRequestCacheStats(Client client, String index) {
return client.admin().indices().prepareStats(index).setRequestCache(true).get().getTotal().getRequestCache();
}
}

0 comments on commit 1e4927b

Please sign in to comment.