Skip to content

Commit

Permalink
Merge branch 'main' into kp/integration-test-stalekeys
Browse files Browse the repository at this point in the history 
Signed-off-by: Kiran Prakash <[email protected]>
  • Loading branch information
@kiranprakash154
kiranprakash154 authored Apr 12, 2024
2 parents 46622fe + c168e1c commit e3ce661
Show file tree
Hide file tree
Showing 81 changed files with 3,549 additions and 625 deletions.
4 changes: 3 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- [Concurrent Segment Search] Perform buildAggregation concurrently and support Composite Aggregations ([#12697](https://github.com/opensearch-project/OpenSearch/pull/12697))
- [Concurrent Segment Search] Disable concurrent segment search for system indices and throttled requests ([#12954](https://github.com/opensearch-project/OpenSearch/pull/12954))
- [Tiered caching] Make Indices Request Cache Stale Key Mgmt Threshold setting dynamic ([#12941](https://github.com/opensearch-project/OpenSearch/pull/12941))
- [Tiered Caching] Make took time caching policy setting dynamic ([#13063](https://github.com/opensearch-project/OpenSearch/pull/13063))
- Derived fields support to derive field values at query time without indexing ([#12569](https://github.com/opensearch-project/OpenSearch/pull/12569))
- Detect breaking changes on pull requests ([#9044](https://github.com/opensearch-project/OpenSearch/pull/9044))
- Add cluster primary balance contraint for rebalancing with buffer ([#12656](https://github.com/opensearch-project/OpenSearch/pull/12656))
- [Remote Store] Make translog transfer timeout configurable ([#12704](https://github.com/opensearch-project/OpenSearch/pull/12704))
- Reject Resize index requests (i.e, split, shrink and clone), While DocRep to SegRep migration is in progress.([#12686](https://github.com/opensearch-project/OpenSearch/pull/12686))
- Add support for more than one protocol for transport ([#12967](https://github.com/opensearch-project/OpenSearch/pull/12967))
- Add changes for overriding remote store and replication settings during snapshot restore. ([#11868](https://github.com/opensearch-project/OpenSearch/pull/11868))

### Dependencies
- Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))
- Bump `asm` from 9.6 to 9.7 ([#12908](https://github.com/opensearch-project/OpenSearch/pull/12908))
- Bump `net.minidev:json-smart` from 2.5.0 to 2.5.1 ([#12893](https://github.com/opensearch-project/OpenSearch/pull/12893))
- Bump `net.minidev:json-smart` from 2.5.0 to 2.5.1 ([#12893](https://github.com/opensearch-project/OpenSearch/pull/12893), [#13117](https://github.com/opensearch-project/OpenSearch/pull/13117))
- Bump `netty` from 4.1.107.Final to 4.1.108.Final ([#12924](https://github.com/opensearch-project/OpenSearch/pull/12924))
- Bump `commons-io:commons-io` from 2.15.1 to 2.16.0 ([#12996](https://github.com/opensearch-project/OpenSearch/pull/12996), [#12998](https://github.com/opensearch-project/OpenSearch/pull/12998), [#12999](https://github.com/opensearch-project/OpenSearch/pull/12999))
- Bump `org.apache.commons:commons-compress` from 1.24.0 to 1.26.1 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
Expand Down
229 changes: 229 additions & 0 deletions client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,19 @@
import org.opensearch.action.search.SearchScrollRequest;
import org.opensearch.client.core.CountRequest;
import org.opensearch.client.core.CountResponse;
import org.opensearch.common.geo.ShapeRelation;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.geometry.Rectangle;
import org.opensearch.index.query.GeoShapeQueryBuilder;
import org.opensearch.index.query.MatchQueryBuilder;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.index.query.RangeQueryBuilder;
import org.opensearch.index.query.ScriptQueryBuilder;
import org.opensearch.index.query.TermsQueryBuilder;
import org.opensearch.join.aggregations.Children;
Expand Down Expand Up @@ -102,6 +106,8 @@
import org.opensearch.search.suggest.Suggest;
import org.opensearch.search.suggest.SuggestBuilder;
import org.opensearch.search.suggest.phrase.PhraseSuggestionBuilder;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.hamcrest.Matchers;
import org.junit.Before;

Expand All @@ -116,6 +122,7 @@
import java.util.concurrent.TimeUnit;

import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.opensearch.index.query.QueryBuilders.geoShapeQuery;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent;
import static org.hamcrest.Matchers.arrayContaining;
import static org.hamcrest.Matchers.both;
Expand Down Expand Up @@ -764,6 +771,228 @@ public void testSearchWithWeirdScriptFields() throws Exception {
}
}

public void testSearchWithDerivedFields() throws Exception {
// Just testing DerivedField definition from SearchSourceBuilder derivedField()
// We are not testing the full functionality here
Request doc = new Request("PUT", "test/_doc/1");
doc.setJsonEntity("{\"field\":\"value\"}");
client().performRequest(doc);
client().performRequest(new Request("POST", "/test/_refresh"));
// Keyword field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "keyword", new Script("emit(params._source[\"field\"])"))
.fetchField("result")
.query(new TermsQueryBuilder("result", "value"))
);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals("value", values.get(0));

// multi valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField(
"result",
"keyword",
new Script("emit(params._source[\"field\"]);emit(params._source[\"field\"] + \"_2\")")
)
.query(new TermsQueryBuilder("result", "value_2"))
.fetchField("result")
);
searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals("value", values.get(0));
assertEquals("value_2", values.get(1));
}
// Boolean field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "boolean", new Script("emit(((String)params._source[\"field\"]).equals(\"value\"))"))
.query(new TermsQueryBuilder("result", "true"))
.fetchField("result")
);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(true, values.get(0));
}
// Long field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "long", new Script("emit(Long.MAX_VALUE)"))
.query(new RangeQueryBuilder("result").from(Long.MAX_VALUE - 1).to(Long.MAX_VALUE))
.fetchField("result")
);

SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(Long.MAX_VALUE, values.get(0));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "long", new Script("emit(Long.MAX_VALUE); emit(Long.MIN_VALUE);"))
.query(new RangeQueryBuilder("result").from(Long.MIN_VALUE).to(Long.MIN_VALUE + 1))
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals(Long.MAX_VALUE, values.get(0));
assertEquals(Long.MIN_VALUE, values.get(1));
}
// Double field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "double", new Script("emit(Double.MAX_VALUE)"))
.query(new RangeQueryBuilder("result").from(Double.MAX_VALUE - 1).to(Double.MAX_VALUE))
.fetchField("result")
);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(Double.MAX_VALUE, values.get(0));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "double", new Script("emit(Double.MAX_VALUE); emit(Double.MIN_VALUE);"))
.query(new RangeQueryBuilder("result").from(Double.MIN_VALUE).to(Double.MIN_VALUE + 1))
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals(Double.MAX_VALUE, values.get(0));
assertEquals(Double.MIN_VALUE, values.get(1));
}
// Date field
{
DateTime date1 = new DateTime(1990, 12, 29, 0, 0, DateTimeZone.UTC);
DateTime date2 = new DateTime(1990, 12, 30, 0, 0, DateTimeZone.UTC);
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "date", new Script("emit(" + date1.getMillis() + "L)"))
.query(new RangeQueryBuilder("result").from(date1.toString()).to(date2.toString()))
.fetchField("result")
);

SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(date1.toString(), values.get(0));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "date", new Script("emit(" + date1.getMillis() + "L); " + "emit(" + date2.getMillis() + "L)"))
.query(new RangeQueryBuilder("result").from(date1.toString()).to(date2.toString()))
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals(date1.toString(), values.get(0));
assertEquals(date2.toString(), values.get(1));
}
// Geo field
{
GeoShapeQueryBuilder qb = geoShapeQuery("result", new Rectangle(-35, 35, 35, -35));
qb.relation(ShapeRelation.INTERSECTS);
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "geo_point", new Script("emit(10.0, 20.0)"))
.query(qb)
.fetchField("result")
);

SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(10.0, ((HashMap) values.get(0)).get("lat"));
assertEquals(20.0, ((HashMap) values.get(0)).get("lon"));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "geo_point", new Script("emit(10.0, 20.0); emit(20.0, 30.0);"))
.query(qb)
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals(10.0, ((HashMap) values.get(0)).get("lat"));
assertEquals(20.0, ((HashMap) values.get(0)).get("lon"));
assertEquals(20.0, ((HashMap) values.get(1)).get("lat"));
assertEquals(30.0, ((HashMap) values.get(1)).get("lon"));
}
// IP field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource().derivedField("result", "ip", new Script("emit(\"10.0.0.1\")")).fetchField("result")
);

SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals("10.0.0.1", values.get(0));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "ip", new Script("emit(\"10.0.0.1\"); emit(\"10.0.0.2\");"))
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals("10.0.0.1", values.get(0));
assertEquals("10.0.0.2", values.get(1));

}

}

public void testSearchScroll() throws Exception {
for (int i = 0; i < 100; i++) {
XContentBuilder builder = jsonBuilder().startObject().field("field", i).endObject();
Expand Down
4 changes: 2 additions & 2 deletions libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,8 @@ public class XContentParserTests extends OpenSearchTestCase {
() -> randomAlphaOfLengthBetween(1, SmileXContent.DEFAULT_MAX_STRING_LEN),
/* YAML parser limitation */
XContentType.YAML,
/* use 75% of the limit, difficult to get the exact size of the content right */
() -> randomRealisticUnicodeOfCodepointLengthBetween(1, (int) (YamlXContent.DEFAULT_CODEPOINT_LIMIT * 0.75))
/* use 50% of the limit, difficult to get the exact size of the content right */
() -> randomRealisticUnicodeOfCodepointLengthBetween(1, (int) (YamlXContent.DEFAULT_CODEPOINT_LIMIT * 0.50))
);

private static final Map<XContentType, Supplier<String>> OFF_LIMIT_GENERATORS = Map.of(
Expand Down
Loading

0 comments on commit e3ce661

Please sign in to comment.