diff --git a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/RankDocsQueryBuilder.java similarity index 90% rename from server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilder.java rename to server/src/main/java/org/elasticsearch/index/query/RankDocsQueryBuilder.java index 1539be9a46ab9..eab88b3a3184d 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/RankDocsQueryBuilder.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.search.retriever.rankdoc; +package org.elasticsearch.index.query; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Query; @@ -16,15 +16,13 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.query.AbstractQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryRewriteContext; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.search.retriever.rankdoc.RankDocsQuery; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.Arrays; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.TransportVersions.RRF_QUERY_REWRITE; @@ -55,6 +53,17 @@ public RankDocsQueryBuilder(StreamInput in) throws IOException { } } + @Override + protected void extractInnerHitBuilders(Map innerHits) { + if (queryBuilders != null) { + for (QueryBuilder query : queryBuilders) { + if (query instanceof AbstractQueryBuilder) { + ((AbstractQueryBuilder) query).extractInnerHitBuilders(innerHits); + } + } + } + } + @Override protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { if (queryBuilders != null) { @@ -71,7 +80,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws return super.doRewrite(queryRewriteContext); } - RankDoc[] rankDocs() { + public RankDoc[] rankDocs() { return rankDocs; } diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 0bb914a9dbf97..1e8410191d99a 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -52,6 +52,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryStringQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.RankDocsQueryBuilder; import org.elasticsearch.index.query.RegexpQueryBuilder; import org.elasticsearch.index.query.ScriptQueryBuilder; import org.elasticsearch.index.query.SimpleQueryStringBuilder; @@ -238,7 +239,6 @@ import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverParserContext; import org.elasticsearch.search.retriever.StandardRetrieverBuilder; -import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.GeoDistanceSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 9c96319136007..b361b9199d43d 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -214,6 +214,8 @@ public static HighlightBuilder highlight() { private Map runtimeMappings = emptyMap(); + private boolean innerHitsDisabled = false; + /** * Constructs a new search source builder. */ @@ -1838,6 +1840,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public void innerHitsDisabled(boolean innerHitsDisabled) { + this.innerHitsDisabled = innerHitsDisabled; + } + + public boolean innerHitsDisabled() { + return this.innerHitsDisabled; + } + public static class IndexBoost implements Writeable, ToXContentObject { private final String index; private final float boost; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java index 0bbbff3a5d5f4..1f968fdf40dd0 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java @@ -265,6 +265,10 @@ public SourceLoader sourceLoader() { return sourceLoader; } + public boolean innerHitsDisabled() { + return searchContext.innerHitsDisabled(); + } + /** * For a hit document that's being processed, return the source lookup representing the * root document. This method is used to pass down the root source when processing this diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java index 374c96fdefe86..6f138c459cef0 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java @@ -40,7 +40,7 @@ public InnerHitsPhase(FetchPhase fetchPhase) { @Override public FetchSubPhaseProcessor getProcessor(FetchContext searchContext) { - if (searchContext.innerHits() == null || searchContext.innerHits().getInnerHits().isEmpty()) { + if (searchContext.innerHitsDisabled() || searchContext.innerHits() == null || searchContext.innerHits().getInnerHits().isEmpty()) { return null; } Map innerHits = searchContext.innerHits().getInnerHits(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 49e41559702ea..dff73f0ae5d9d 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -402,4 +402,8 @@ public String toString() { public abstract SourceLoader newSourceLoader(); public abstract IdLoader newIdLoader(); + + public boolean innerHitsDisabled() { + return request() != null && request().innerHitsDisabled(); + } } diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index fcfe6fbb70030..1ad5848650aa2 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -82,6 +82,7 @@ public class ShardSearchRequest extends TransportRequest implements IndicesReque private Boolean requestCache; private final long nowInMillis; private final boolean allowPartialSearchResults; + public final boolean innerHitsDisabled; private final OriginalIndices originalIndices; private boolean canReturnNullResponseIfMatchNoDocs; @@ -250,6 +251,7 @@ public ShardSearchRequest( this.waitForCheckpoint = waitForCheckpoint; this.waitForCheckpointsTimeout = waitForCheckpointsTimeout; this.forceSyntheticSource = forceSyntheticSource; + this.innerHitsDisabled = source.innerHitsDisabled(); } @SuppressWarnings("this-escape") @@ -275,6 +277,7 @@ public ShardSearchRequest(ShardSearchRequest clone) { this.waitForCheckpoint = clone.waitForCheckpoint; this.waitForCheckpointsTimeout = clone.waitForCheckpointsTimeout; this.forceSyntheticSource = clone.forceSyntheticSource; + this.innerHitsDisabled = clone.innerHitsDisabled; } public ShardSearchRequest(StreamInput in) throws IOException { @@ -341,6 +344,11 @@ public ShardSearchRequest(StreamInput in) throws IOException { */ forceSyntheticSource = false; } + if (in.getTransportVersion().onOrAfter(TransportVersion.current())) { + innerHitsDisabled = in.readBoolean(); + } else { + innerHitsDisabled = false; + } originalIndices = OriginalIndices.readOriginalIndices(in); } @@ -401,6 +409,9 @@ protected final void innerWriteTo(StreamOutput out, boolean asKey) throws IOExce throw new IllegalArgumentException("force_synthetic_source is not supported before 8.4.0"); } } + if (out.getTransportVersion().onOrAfter(TransportVersion.current())) { + out.writeBoolean(innerHitsDisabled); + } } @Override @@ -482,6 +493,10 @@ public boolean allowPartialSearchResults() { return allowPartialSearchResults; } + public boolean innerHitsDisabled() { + return innerHitsDisabled; + } + public Scroll scroll() { return scroll; } diff --git a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java index b15798db95b6f..6fcf818459024 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java @@ -254,6 +254,7 @@ protected SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, } sortBuilders.add(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME)); sourceBuilder.sort(sortBuilders); + sourceBuilder.innerHitsDisabled(true); return sourceBuilder; } diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index facda1a30a5ac..264a231b6ce25 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -15,8 +15,8 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.RankDocsQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; import org.elasticsearch.search.vectors.ExactKnnQueryBuilder; import org.elasticsearch.search.vectors.KnnSearchBuilder; import org.elasticsearch.search.vectors.QueryVectorBuilder; @@ -240,6 +240,9 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder List knnSearchBuilders = new ArrayList<>(searchSourceBuilder.knnSearch()); knnSearchBuilders.add(knnSearchBuilder); searchSourceBuilder.knnSearch(knnSearchBuilders); + if (compoundUsed) { + searchSourceBuilder.innerHitsDisabled(true); + } } // ---- FOR TESTING XCONTENT PARSING ---- diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java index 535db5c8fe28e..02f890f51d011 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java @@ -12,9 +12,9 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.RankDocsQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rank.RankDoc; -import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; diff --git a/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java index 4e875a97fdfc4..f1215a4dfb983 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java @@ -201,6 +201,9 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder if (collapseBuilder != null) { searchSourceBuilder.collapse(collapseBuilder); } + if (compoundUsed) { + searchSourceBuilder.innerHitsDisabled(true); + } } // ---- FOR TESTING XCONTENT PARSING ---- diff --git a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java index 2cb960e7e73cb..ebbdf58cc8c4f 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java @@ -283,7 +283,7 @@ private static int[] findSegmentStarts(IndexReader reader, RankDoc[] docs) { return starts; } - RankDoc[] rankDocs() { + public RankDoc[] rankDocs() { return docs; } diff --git a/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/RankDocsQueryBuilderTests.java similarity index 98% rename from server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java rename to server/src/test/java/org/elasticsearch/index/query/RankDocsQueryBuilderTests.java index e8f88f3297b78..ba39702d3d162 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RankDocsQueryBuilderTests.java @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.search.retriever.rankdoc; +package org.elasticsearch.index.query; import org.apache.lucene.document.Document; import org.apache.lucene.document.NumericDocValuesField; @@ -22,9 +22,8 @@ import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.search.retriever.rankdoc.RankDocsQuery; import org.elasticsearch.test.AbstractQueryTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/elasticsearch/search/rank/AbstractRankDocWireSerializingTestCase.java b/server/src/test/java/org/elasticsearch/search/rank/AbstractRankDocWireSerializingTestCase.java index d0c85a33acf09..8cc40570ab4bb 100644 --- a/server/src/test/java/org/elasticsearch/search/rank/AbstractRankDocWireSerializingTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/rank/AbstractRankDocWireSerializingTestCase.java @@ -12,8 +12,8 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RankDocsQueryBuilder; import org.elasticsearch.search.SearchModule; -import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java index b0bf7e6636498..7923cb5f0d918 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java @@ -17,11 +17,11 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.RandomQueryBuilder; +import org.elasticsearch.index.query.RankDocsQueryBuilder; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rank.RankDoc; -import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.usage.SearchUsage; import org.elasticsearch.xcontent.NamedXContentRegistry; diff --git a/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java b/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java index 384564ac01e2a..af6782c45dce8 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java @@ -13,11 +13,11 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.RandomQueryBuilder; +import org.elasticsearch.index.query.RankDocsQueryBuilder; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rank.RankDoc; -import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; import org.elasticsearch.test.ESTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilder.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilder.java index 9ef2f630b50bd..ef02bc02d212e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilder.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilder.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RankDocsQueryBuilder; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -19,7 +20,6 @@ import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverBuilderWrapper; import org.elasticsearch.search.retriever.RetrieverParserContext; -import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.xcontent.ConstructingObjectParser; diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml index f3914843b80ec..4389321373984 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml @@ -35,6 +35,11 @@ setup: properties: views: type: long + nested_inner_hits: + type: nested + properties: + data: + type: keyword - do: index: @@ -125,6 +130,7 @@ setup: integer: 2 keyword: "technology" nested: { views: 10} + nested_inner_hits: [{"data": "foo"}, {"data": "bar"}, {"data": "baz"}] - do: indices.refresh: {} @@ -960,3 +966,55 @@ setup: - length: { hits.hits : 1 } - match: { hits.hits.0._id: "1" } + +--- +"rrf retriever with inner_hits for sub-retriever": + + - do: + search: + _source: false + index: test + body: + retriever: + rrf: + retrievers: [ + { + standard: { + query: { + nested: { + path: nested_inner_hits, + inner_hits: { + _source: false, + "sort": [ { + "nested_inner_hits.data": "asc" + } + ], + fields: [ nested_inner_hits.data ] + }, + query: { + match_all: { } + } + } + } + } + }, + { + standard: { + query: { + match_all: { } + } + } + } + ] + rank_window_size: 10 + rank_constant: 10 + size: 2 + + - match: { hits.total.value: 9 } + + - match: { hits.hits.0.inner_hits.nested_inner_hits.hits.total.value: 3 } + - match: { hits.hits.0.inner_hits.nested_inner_hits.hits.hits.0.fields.nested_inner_hits.0.data.0: bar } + - match: { hits.hits.0.inner_hits.nested_inner_hits.hits.hits.1.fields.nested_inner_hits.0.data.0: baz } + - match: { hits.hits.0.inner_hits.nested_inner_hits.hits.hits.2.fields.nested_inner_hits.0.data.0: foo } + + - match: { hits.hits.1.inner_hits.nested_inner_hits.hits.total.value: 0 }