From e9ad245012182f556a9ef78635844aa4adc02153 Mon Sep 17 00:00:00 2001 From: Ethan Emoto Date: Tue, 7 Nov 2023 17:42:39 -0800 Subject: [PATCH] Fixed test cases for concurrent search Fixed test cases Updated fix for TransportTwoNodesSearchIT test cases Updated test cases and muted flaky tests Move indexRandomForConcurrentSearch call into prepareData Remove changes from TransportTwoNodesSearchIT.java until future fix Signed-off-by: Ethan Emoto --- .../search/aggregations/pipeline/DerivativeIT.java | 1 + .../search/aggregations/pipeline/MovAvgIT.java | 1 + .../search/basic/SearchRedStateIndexIT.java | 1 + .../search/basic/SearchWhileCreatingIndexIT.java | 1 + .../search/fetch/FetchSubPhasePluginIT.java | 5 +++++ .../functionscore/RandomScoreFunctionIT.java | 3 +++ .../org/opensearch/search/geo/GeoFilterIT.java | 2 ++ .../search/morelikethis/MoreLikeThisIT.java | 14 +++++++++++++- 8 files changed, 27 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java index 18484c8a60ed7..41bbffc13658b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/DerivativeIT.java @@ -693,6 +693,7 @@ public void testAvgMovavgDerivNPE() throws Exception { } refresh(); + indexRandomForConcurrentSearch("movavg_npe"); SearchResponse response = client().prepareSearch("movavg_npe") .addAggregation( diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java index 2aad0d2d38901..8ad3107ac33ac 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/pipeline/MovAvgIT.java @@ -1320,6 +1320,7 @@ public void testPredictWithNonEmptyBuckets() throws Exception { .setSource(jsonBuilder().startObject().field(INTERVAL_FIELD, i).field(VALUE_FIELD2, 10).endObject()) ); } + indexRandomForConcurrentSearch("predict_non_empty"); bulkBuilder.get(); ensureSearchable(); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java index 44c4981dfdb36..bd623ccdf2731 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchRedStateIndexIT.java @@ -153,6 +153,7 @@ private void buildRedIndex(int numShards) throws Exception { client().prepareIndex("test").setId("" + i).setSource("field1", "value1").get(); } refresh(); + indexRandomForConcurrentSearch("test"); internalCluster().stopRandomDataNode(); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileCreatingIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileCreatingIndexIT.java index 71af7215c4eb7..a5989b693d332 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileCreatingIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/basic/SearchWhileCreatingIndexIT.java @@ -106,6 +106,7 @@ private void searchWhileCreatingIndex(boolean createIndex, int numberOfReplicas) } client().prepareIndex("test").setId(id).setSource("field", "test").get(); RefreshResponse refreshResponse = client().admin().indices().prepareRefresh("test").get(); + indexRandomForConcurrentSearch("test"); // at least one shard should be successful when refreshing assertThat(refreshResponse.getSuccessfulShards(), greaterThanOrEqualTo(1)); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java index 86df25c4dad65..77ddc5bf636f0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fetch/FetchSubPhasePluginIT.java @@ -100,6 +100,10 @@ protected Collection> nodePlugins() { @SuppressWarnings("unchecked") public void testPlugin() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11112", + internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING) + ); client().admin() .indices() .prepareCreate("test") @@ -119,6 +123,7 @@ public void testPlugin() throws Exception { .actionGet(); client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch() .setSource(new SearchSourceBuilder().ext(Collections.singletonList(new TermVectorsFetchBuilder("test")))) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java index 8f43cefd2d53b..54cfcb7e6a683 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/RandomScoreFunctionIT.java @@ -207,6 +207,7 @@ public void testScoreAccessWithinScript() throws Exception { .get(); } refresh(); + indexRandomForConcurrentSearch("test"); Map params = new HashMap<>(); params.put("factor", randomIntBetween(2, 4)); @@ -368,6 +369,7 @@ public void testScoreRange() throws Exception { } flush(); refresh(); + indexRandomForConcurrentSearch("test"); int iters = scaledRandomIntBetween(10, 20); for (int i = 0; i < iters; ++i) { SearchResponse searchResponse = client().prepareSearch() @@ -390,6 +392,7 @@ public void testSeeds() throws Exception { index("test", "type", "" + i, jsonBuilder().startObject().endObject()); } flushAndRefresh(); + indexRandomForConcurrentSearch("test"); assertNoFailures( client().prepareSearch() diff --git a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java index 00524c6e04707..ba519be04edff 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/geo/GeoFilterIT.java @@ -266,6 +266,7 @@ public void testShapeRelations() throws Exception { client().prepareIndex("shapes").setId("1").setSource(data, MediaTypeRegistry.JSON).get(); client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("shapes"); // Point in polygon SearchResponse result = client().prepareSearch() @@ -427,6 +428,7 @@ public void testBulk() throws Exception { client().admin().indices().prepareCreate("countries").setSettings(settings).setMapping(xContentBuilder).get(); BulkResponse bulk = client().prepareBulk().add(bulkAction, 0, bulkAction.length, null, xContentBuilder.contentType()).get(); + indexRandomForConcurrentSearch("countries"); for (BulkItemResponse item : bulk.getItems()) { assertFalse("unable to index data", item.isFailed()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/morelikethis/MoreLikeThisIT.java b/server/src/internalClusterTest/java/org/opensearch/search/morelikethis/MoreLikeThisIT.java index 87435bb0bd09d..dc7c4e687c2fa 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/morelikethis/MoreLikeThisIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/morelikethis/MoreLikeThisIT.java @@ -125,6 +125,7 @@ public void testSimpleMoreLikeThis() throws Exception { client().index(indexRequest("test").id("2").source(jsonBuilder().startObject().field("text", "lucene release").endObject())) .actionGet(); client().admin().indices().refresh(refreshRequest()).actionGet(); + indexRandomForConcurrentSearch("test"); logger.info("Running moreLikeThis"); SearchResponse response = client().prepareSearch() @@ -155,6 +156,7 @@ public void testSimpleMoreLikeThisWithTypes() throws Exception { client().index(indexRequest("test").id("2").source(jsonBuilder().startObject().field("text", "lucene release").endObject())) .actionGet(); client().admin().indices().refresh(refreshRequest()).actionGet(); + indexRandomForConcurrentSearch("test"); logger.info("Running moreLikeThis"); SearchResponse response = client().prepareSearch() @@ -190,6 +192,7 @@ public void testMoreLikeThisForZeroTokensInOneOfTheAnalyzedFields() throws Excep ).actionGet(); client().admin().indices().refresh(refreshRequest()).actionGet(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setQuery( @@ -256,6 +259,7 @@ public void testMoreLikeThisWithAliases() throws Exception { client().index(indexRequest("test").id("4").source(jsonBuilder().startObject().field("text", "opensearch release").endObject())) .actionGet(); client().admin().indices().refresh(refreshRequest()).actionGet(); + indexRandomForConcurrentSearch("test"); logger.info("Running moreLikeThis on index"); SearchResponse response = client().prepareSearch() @@ -304,6 +308,7 @@ public void testMoreLikeThisWithAliasesInLikeDocuments() throws Exception { client().index(indexRequest(indexName).id("3").source(jsonBuilder().startObject().field("text", "opensearch index").endObject())) .actionGet(); refresh(indexName); + indexRandomForConcurrentSearch(indexName); SearchResponse response = client().prepareSearch() .setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item(aliasName, "1") }).minTermFreq(1).minDocFreq(1)) @@ -321,6 +326,7 @@ public void testMoreLikeThisIssue2197() throws Exception { .get(); client().admin().indices().prepareRefresh("foo").get(); assertThat(ensureGreen(), equalTo(ClusterHealthStatus.GREEN)); + indexRandomForConcurrentSearch("foo"); SearchResponse response = client().prepareSearch() .setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1") })) @@ -344,6 +350,7 @@ public void testMoreLikeWithCustomRouting() throws Exception { .setRouting("2") .get(); client().admin().indices().prepareRefresh("foo").get(); + indexRandomForConcurrentSearch("foo"); SearchResponse response = client().prepareSearch() .setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1").routing("2") })) @@ -368,6 +375,7 @@ public void testMoreLikeThisIssueRoutingNotSerialized() throws Exception { .setRouting("4000") .get(); client().admin().indices().prepareRefresh("foo").get(); + indexRandomForConcurrentSearch("foo"); SearchResponse response = client().prepareSearch() .setQuery(new MoreLikeThisQueryBuilder(null, new Item[] { new Item("foo", "1").routing("4000") })) .get(); @@ -401,6 +409,7 @@ public void testNumericField() throws Exception { .get(); refresh(); + indexRandomForConcurrentSearch("test"); // Implicit list of fields -> ignore numeric fields SearchResponse searchResponse = client().prepareSearch() @@ -508,6 +517,7 @@ public void testMoreLikeThisWithFieldAlias() throws Exception { index("test", "_doc", "1", "text", "lucene"); index("test", "_doc", "2", "text", "lucene release"); refresh(); + indexRandomForConcurrentSearch("test"); Item item = new Item("test", "1"); QueryBuilder query = QueryBuilders.moreLikeThisQuery(new String[] { "alias" }, null, new Item[] { item }) @@ -548,6 +558,7 @@ public void testSimpleMoreLikeInclude() throws Exception { .source(jsonBuilder().startObject().field("text", "Lucene has been ported to other programming languages").endObject()) ).actionGet(); client().admin().indices().refresh(refreshRequest()).actionGet(); + indexRandomForConcurrentSearch("test"); logger.info("Running More Like This with include true"); SearchResponse response = client().prepareSearch() @@ -832,11 +843,12 @@ public void testSelectFields() throws IOException, ExecutionException, Interrupt assertHitCount(response, 1); } - public void testWithRouting() throws IOException { + public void testWithRouting() throws IOException, InterruptedException { client().prepareIndex("index").setId("1").setRouting("3").setSource("text", "this is a document").get(); client().prepareIndex("index").setId("2").setRouting("1").setSource("text", "this is another document").get(); client().prepareIndex("index").setId("3").setRouting("4").setSource("text", "this is yet another document").get(); refresh("index"); + indexRandomForConcurrentSearch("index"); Item item = new Item("index", "2").routing("1"); MoreLikeThisQueryBuilder moreLikeThisQueryBuilder = new MoreLikeThisQueryBuilder(