Skip to content

Commit

Permalink
Add indexRandomForConcurrentSearch to tests (#11259) (#11265)
Browse files Browse the repository at this point in the history 
* Update Concurrent search related tests to use slice count > 1.




* Add indexRandomForConcurrentSearch to tests:
* CardinalityWithRequestBreakerIT.java
* TopHitsIT.java
* SearchFieldsIT.java
* DecayFunctionScoreIT.java
* FunctionScoreFieldValueIT.java
* FunctionScoreIT.java
* FunctionScorePluginIT.java
* QueryRescorerIT.java



---------




(cherry picked from commit edbc4e2)

Signed-off-by: Harish Bhakuni <[email protected]>
Signed-off-by: Jay Deng <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Harish Bhakuni <[email protected]>
  • Loading branch information
@opensearch-trigger-bot @github-actions
3 people authored Nov 20, 2023
1 parent f2cdd9f commit 99e9c83
Show file tree
Hide file tree
Showing 7 changed files with 48 additions and 8 deletions.
1 change: 1 addition & 0 deletions ...Test/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ public void testRequestBreaker() throws Exception {
)
.get();

indexRandomForConcurrentSearch("test");
try {
client().prepareSearch("test")
.addAggregation(
Expand Down
16 changes: 16 additions & 0 deletions server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,7 @@ public void testStoredFields() throws Exception {
.get();

client().admin().indices().prepareRefresh().get();
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field1").get();
assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
Expand Down Expand Up @@ -358,6 +359,7 @@ public void testScriptDocAndFields() throws Exception {
)
.get();
client().admin().indices().refresh(refreshRequest()).actionGet();
indexRandomForConcurrentSearch("test");

logger.info("running doc['num1'].value");
SearchResponse response = client().prepareSearch()
Expand Down Expand Up @@ -458,6 +460,7 @@ public void testScriptWithUnsignedLong() throws Exception {
)
.get();
client().admin().indices().refresh(refreshRequest()).actionGet();
indexRandomForConcurrentSearch("test");

SearchResponse response = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -547,6 +550,7 @@ public void testScriptFieldWithNanos() throws Exception {
.setSource(jsonBuilder().startObject().field("date", "1970-01-01T00:00:00.000Z").endObject()),
client().prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("date", date).endObject())
);
indexRandomForConcurrentSearch("test");

SearchResponse response = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -632,6 +636,7 @@ public void testScriptFieldUsingSource() throws Exception {
)
.get();
client().admin().indices().refresh(refreshRequest()).actionGet();
indexRandomForConcurrentSearch("test");

SearchResponse response = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -674,6 +679,7 @@ public void testScriptFieldUsingSource() throws Exception {
public void testScriptFieldsForNullReturn() throws Exception {
client().prepareIndex("test").setId("1").setSource("foo", "bar").setRefreshPolicy("true").get();

indexRandomForConcurrentSearch("test");
SearchResponse response = client().prepareSearch()
.setQuery(matchAllQuery())
.addScriptField("test_script_1", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "return null", Collections.emptyMap()))
Expand Down Expand Up @@ -795,6 +801,7 @@ public void testStoredFieldsWithoutSource() throws Exception {
.get();

client().admin().indices().prepareRefresh().get();
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -852,6 +859,7 @@ public void testSearchFieldsMetadata() throws Exception {
.setSource(jsonBuilder().startObject().field("field1", "value").endObject())
.setRefreshPolicy(IMMEDIATE)
.get();
indexRandomForConcurrentSearch("my-index");

SearchResponse searchResponse = client().prepareSearch("my-index").addStoredField("field1").addStoredField("_routing").get();

Expand All @@ -866,6 +874,7 @@ public void testSearchFieldsNonLeafField() throws Exception {
.setSource(jsonBuilder().startObject().startObject("field1").field("field2", "value1").endObject().endObject())
.setRefreshPolicy(IMMEDIATE)
.get();
indexRandomForConcurrentSearch("my-index");

assertFailures(
client().prepareSearch("my-index").addStoredField("field1"),
Expand Down Expand Up @@ -932,6 +941,7 @@ public void testGetFieldsComplexField() throws Exception {
);

client().prepareIndex("my-index").setId("1").setRefreshPolicy(IMMEDIATE).setSource(source, MediaTypeRegistry.JSON).get();
indexRandomForConcurrentSearch("my-index");

String field = "field1.field2.field3.field4";

Expand Down Expand Up @@ -1039,6 +1049,7 @@ public void testDocValueFields() throws Exception {
.get();

client().admin().indices().prepareRefresh().get();
indexRandomForConcurrentSearch("test");

SearchRequestBuilder builder = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -1271,6 +1282,7 @@ public void testScriptFields() throws Exception {
);
}
indexRandom(true, reqs);
indexRandomForConcurrentSearch("index");
ensureSearchable();
SearchRequestBuilder req = client().prepareSearch("index");
for (String field : Arrays.asList("s", "ms", "l", "ml", "d", "md")) {
Expand Down Expand Up @@ -1326,6 +1338,7 @@ public void testDocValueFieldsWithFieldAlias() throws Exception {

index("test", MapperService.SINGLE_MAPPING_NAME, "1", "text_field", "foo", "date_field", formatter.print(date));
refresh("test");
indexRandomForConcurrentSearch("test");

SearchRequestBuilder builder = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -1387,6 +1400,7 @@ public void testWildcardDocValueFieldsWithFieldAlias() throws Exception {

index("test", MapperService.SINGLE_MAPPING_NAME, "1", "text_field", "foo", "date_field", formatter.print(date));
refresh("test");
indexRandomForConcurrentSearch("test");

SearchRequestBuilder builder = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -1440,6 +1454,7 @@ public void testStoredFieldsWithFieldAlias() throws Exception {

index("test", MapperService.SINGLE_MAPPING_NAME, "1", "field1", "value1", "field2", "value2");
refresh("test");
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch()
.setQuery(matchAllQuery())
Expand Down Expand Up @@ -1482,6 +1497,7 @@ public void testWildcardStoredFieldsWithFieldAlias() throws Exception {

index("test", MapperService.SINGLE_MAPPING_NAME, "1", "field1", "value1", "field2", "value2");
refresh("test");
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field*").get();
assertHitCount(searchResponse, 1L);
Expand Down
10 changes: 10 additions & 0 deletions ...rc/internalClusterTest/java/org/opensearch/search/functionscore/DecayFunctionScoreIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,7 @@ public void testBoostModeSettingWorks() throws Exception {
)
);
indexRandom(true, false, indexBuilders); // force no dummy docs
indexRandomForConcurrentSearch("test");

// Test Gauss
List<Float> lonlat = new ArrayList<>();
Expand Down Expand Up @@ -482,6 +483,7 @@ public void testParseGeoPoint() throws Exception {
constantScoreQuery(termQuery("test", "value")),
ScoreFunctionBuilders.weightFactorFunction(randomIntBetween(1, 10))
);
indexRandomForConcurrentSearch("test");
GeoPoint point = new GeoPoint(20, 11);
ActionFuture<SearchResponse> response = client().search(
searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
Expand Down Expand Up @@ -535,6 +537,7 @@ public void testCombineModes() throws Exception {
.setRefreshPolicy(IMMEDIATE)
.setSource(jsonBuilder().startObject().field("test", "value value").field("num", 1.0).endObject())
.get();
indexRandomForConcurrentSearch("test");
FunctionScoreQueryBuilder baseQuery = functionScoreQuery(
constantScoreQuery(termQuery("test", "value")),
ScoreFunctionBuilders.weightFactorFunction(2)
Expand Down Expand Up @@ -654,6 +657,7 @@ public void testCombineModesExplain() throws Exception {
constantScoreQuery(termQuery("test", "value")).queryName("query1"),
ScoreFunctionBuilders.weightFactorFunction(2, "weight1")
);
indexRandomForConcurrentSearch("test");
// decay score should return 0.5 for this function and baseQuery should return 2.0f as it's score
ActionFuture<SearchResponse> response = client().search(
searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
Expand Down Expand Up @@ -762,6 +766,7 @@ public void testParseDateMath() throws Exception {
).actionGet();
refresh();

indexRandomForConcurrentSearch("test");
SearchResponse sr = client().search(
searchRequest().source(
searchSource().query(functionScoreQuery(termQuery("test", "value"), gaussDecayFunction("num1", "now", "2d")))
Expand Down Expand Up @@ -817,6 +822,7 @@ public void testValueMissingLin() throws Exception {
).actionGet();

refresh();
indexRandomForConcurrentSearch("test");

ActionFuture<SearchResponse> response = client().search(
searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
Expand Down Expand Up @@ -893,6 +899,7 @@ public void testDateWithoutOrigin() throws Exception {
).actionGet();

refresh();
indexRandomForConcurrentSearch("test");

ActionFuture<SearchResponse> response = client().search(
searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
Expand Down Expand Up @@ -974,6 +981,7 @@ public void testManyDocsLin() throws Exception {
List<Float> lonlat = new ArrayList<>();
lonlat.add(100f);
lonlat.add(110f);
indexRandomForConcurrentSearch("test");
ActionFuture<SearchResponse> response = client().search(
searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
.source(
Expand Down Expand Up @@ -1107,6 +1115,7 @@ public void testNoQueryGiven() throws Exception {
client().index(indexRequest("test").source(jsonBuilder().startObject().field("test", "value").field("num", 1.0).endObject()))
.actionGet();
refresh();
indexRandomForConcurrentSearch("test");
// so, we indexed a string field, but now we try to score a num field
ActionFuture<SearchResponse> response = client().search(
searchRequest().searchType(SearchType.QUERY_THEN_FETCH)
Expand Down Expand Up @@ -1171,6 +1180,7 @@ public void testMultiFieldOptions() throws Exception {
);

indexRandom(true, doc1, doc2);
indexRandomForConcurrentSearch("test");

ActionFuture<SearchResponse> response = client().search(searchRequest().source(searchSource().query(baseQuery)));
SearchResponse sr = response.actionGet();
Expand Down
7 changes: 4 additions & 3 deletions ...ternalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreFieldValueIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ protected Settings featureFlagSettings() {
return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
}

public void testFieldValueFactor() throws IOException {
public void testFieldValueFactor() throws IOException, InterruptedException {
assertAcked(
prepareCreate("test").setMapping(
jsonBuilder().startObject()
Expand All @@ -99,8 +99,8 @@ public void testFieldValueFactor() throws IOException {
client().prepareIndex("test").setId("1").setSource("test", 5, "body", "foo").get();
client().prepareIndex("test").setId("2").setSource("test", 17, "body", "foo").get();
client().prepareIndex("test").setId("3").setSource("body", "bar").get();

refresh();
indexRandomForConcurrentSearch("test");

// document 2 scores higher because 17 > 5
SearchResponse response = client().prepareSearch("test")
Expand Down Expand Up @@ -189,7 +189,7 @@ public void testFieldValueFactor() throws IOException {
}
}

public void testFieldValueFactorExplain() throws IOException {
public void testFieldValueFactorExplain() throws IOException, InterruptedException {
assertAcked(
prepareCreate("test").setMapping(
jsonBuilder().startObject()
Expand All @@ -210,6 +210,7 @@ public void testFieldValueFactorExplain() throws IOException {
client().prepareIndex("test").setId("3").setSource("body", "bar").get();

refresh();
indexRandomForConcurrentSearch("test");

// document 2 scores higher because 17 > 5
final String functionName = "func1";
Expand Down
13 changes: 9 additions & 4 deletions server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,11 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
}
}

public void testScriptScoresNested() throws IOException {
public void testScriptScoresNested() throws IOException, InterruptedException {
createIndex(INDEX);
index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
refresh();
indexRandomForConcurrentSearch(INDEX);

Script scriptOne = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "1", Collections.emptyMap());
Script scriptTwo = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap());
Expand All @@ -148,10 +149,11 @@ public void testScriptScoresNested() throws IOException {
assertThat(response.getHits().getAt(0).getScore(), equalTo(1.0f));
}

public void testScriptScoresWithAgg() throws IOException {
public void testScriptScoresWithAgg() throws IOException, InterruptedException {
createIndex(INDEX);
index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
refresh();
indexRandomForConcurrentSearch(INDEX);

Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap());

Expand All @@ -166,10 +168,11 @@ public void testScriptScoresWithAgg() throws IOException {
assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), is(1L));
}

public void testScriptScoresWithAggWithExplain() throws IOException {
public void testScriptScoresWithAggWithExplain() throws IOException, InterruptedException {
createIndex(INDEX);
index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject());
refresh();
indexRandomForConcurrentSearch(INDEX);

Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap());

Expand All @@ -195,7 +198,7 @@ public void testScriptScoresWithAggWithExplain() throws IOException {
assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), is(1L));
}

public void testMinScoreFunctionScoreBasic() throws IOException {
public void testMinScoreFunctionScoreBasic() throws IOException, InterruptedException {
float score = randomValueOtherThanMany((f) -> Float.compare(f, 0) < 0, OpenSearchTestCase::randomFloat);
float minScore = randomValueOtherThanMany((f) -> Float.compare(f, 0) < 0, OpenSearchTestCase::randomFloat);
index(
Expand All @@ -207,6 +210,7 @@ public void testMinScoreFunctionScoreBasic() throws IOException {
.endObject()
);
refresh();
indexRandomForConcurrentSearch(INDEX);
ensureYellow();

Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['random_score']", Collections.emptyMap());
Expand Down Expand Up @@ -291,6 +295,7 @@ public void testWithEmptyFunctions() throws IOException, ExecutionException, Int
assertAcked(prepareCreate("test"));
index("test", "testtype", "1", jsonBuilder().startObject().field("text", "test text").endObject());
refresh();
indexRandomForConcurrentSearch("test");

SearchResponse termQuery = client().search(searchRequest().source(searchSource().explain(true).query(termQuery("text", "text"))))
.get();
Expand Down
1 change: 1 addition & 0 deletions ...c/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ public void testPlugin() throws Exception {
).actionGet();

client().admin().indices().prepareRefresh().get();
indexRandomForConcurrentSearch("test");
DecayFunctionBuilder<?> gfb = new CustomDistanceScoreBuilder("num1", "2013-05-28", "+1d");

ActionFuture<SearchResponse> response = client().search(
Expand Down
8 changes: 7 additions & 1 deletion server/src/internalClusterTest/java/org/opensearch/search/functionscore/QueryRescorerIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,14 +110,15 @@ protected Settings featureFlagSettings() {
return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
}

public void testEnforceWindowSize() {
public void testEnforceWindowSize() throws InterruptedException {
createIndex("test");
// this
int iters = scaledRandomIntBetween(10, 20);
for (int i = 0; i < iters; i++) {
client().prepareIndex("test").setId(Integer.toString(i)).setSource("f", Integer.toString(i)).get();
}
refresh();
indexRandomForConcurrentSearch("test");

int numShards = getNumShards("test").numPrimaries;
for (int j = 0; j < iters; j++) {
Expand Down Expand Up @@ -169,6 +170,7 @@ public void testRescorePhrase() throws Exception {
.setSource("field1", "quick huge brown", "field2", "the quick lazy huge brown fox jumps over the tree")
.get();
refresh();
indexRandomForConcurrentSearch("test");
SearchResponse searchResponse = client().prepareSearch()
.setQuery(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR))
.setRescorer(
Expand Down Expand Up @@ -474,6 +476,7 @@ private static void assertEquivalent(String query, SearchResponse plain, SearchR
public void testEquivalence() throws Exception {
// no dummy docs since merges can change scores while we run queries.
int numDocs = indexRandomNumbers("whitespace", -1, false);
indexRandomForConcurrentSearch("test");

final int iters = scaledRandomIntBetween(50, 100);
for (int i = 0; i < iters; i++) {
Expand Down Expand Up @@ -545,6 +548,7 @@ public void testExplain() throws Exception {
.setSource("field1", "quick huge brown", "field2", "the quick lazy huge brown fox jumps over the tree")
.get();
refresh();
indexRandomForConcurrentSearch("test");

{
SearchResponse searchResponse = client().prepareSearch()
Expand Down Expand Up @@ -816,6 +820,7 @@ public void testFromSize() throws Exception {
client().prepareIndex("test").setId("" + i).setSource("text", "hello world").get();
}
refresh();
indexRandomForConcurrentSearch("test");

SearchRequestBuilder request = client().prepareSearch();
request.setQuery(QueryBuilders.termQuery("text", "hello"));
Expand All @@ -832,6 +837,7 @@ public void testRescorePhaseWithInvalidSort() throws Exception {
client().prepareIndex("test").setId("" + i).setSource("number", 0).get();
}
refresh();
indexRandomForConcurrentSearch("test");

Exception exc = expectThrows(
Exception.class,
Expand Down

0 comments on commit 99e9c83

Please sign in to comment.