Merge branch 'refs/heads/main' into carlosdelest/fix-synonyms-ci-test…

…s-timeout

docs/changelog/112929.yaml

@@ -0,0 +1,5 @@
+pr: 112929
+summary: "ES|QL: Add support for cached strings in plan serialization"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/112938.yaml

@@ -0,0 +1,35 @@
+pr: 112938
+summary: Enhance SORT push-down to Lucene to cover references to fields and ST_DISTANCE function
+area: ES|QL
+type: enhancement
+issues:
+ - 109973
+highlight:
+  title: Enhance SORT push-down to Lucene to cover references to fields and ST_DISTANCE function
+  body: |-
+    The most used and likely most valuable geospatial search query in Elasticsearch is the sorted proximity search,
+    finding items within a certain distance of a point of interest and sorting the results by distance.
+    This has been possible in ES|QL since 8.15.0, but the sorting was done in-memory, not pushed down to Lucene.
+    Now the sorting is pushed down to Lucene, which results in a significant performance improvement.
+
+    Queries that perform both filtering and sorting on distance are supported. For example:
+
+    [source,esql]
+    ----
+    FROM test
+    | EVAL distance = ST_DISTANCE(location, TO_GEOPOINT("POINT(37.7749, -122.4194)"))
+    | WHERE distance < 1000000
+    | SORT distance ASC, name DESC
+    | LIMIT 10
+    ----
+
+    In addition, the support for sorting on EVAL expressions has been extended to cover references to fields:
+
+    [source,esql]
+    ----
+    FROM test
+    | EVAL ref = field
+    | SORT ref ASC
+    | LIMIT 10
+    ----
+  notable: false

docs/changelog/114411.yaml

@@ -0,0 +1,5 @@
+pr: 114411
+summary: "ESQL: Push down filters even in case of renames in Evals"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/114533.yaml

@@ -0,0 +1,5 @@
+pr: 114533
+summary: Fix dim validation for bit `element_type`
+area: Vector Search
+type: bug
+issues: []

docs/changelog/114552.yaml

@@ -0,0 +1,5 @@
+pr: 114552
+summary: Improve exception message for bad environment variable placeholders in settings
+area: Infra/Settings
+type: enhancement
+issues: [110858]

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/DeleteDataStreamLifecycleAction.java

@@ -34,7 +34,26 @@ private DeleteDataStreamLifecycleAction() {/* no instances */}
     public static final class Request extends AcknowledgedRequest<Request> implements IndicesRequest.Replaceable {
 
         private String[] names;
-        private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false);
+        private IndicesOptions indicesOptions = IndicesOptions.builder()
+            .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+            .wildcardOptions(
+                IndicesOptions.WildcardOptions.builder()
+                    .matchOpen(true)
+                    .matchClosed(true)
+                    .includeHidden(false)
+                    .resolveAliases(false)
+                    .allowEmptyExpressions(true)
+                    .build()
+            )
+            .gatekeeperOptions(
+                IndicesOptions.GatekeeperOptions.builder()
+                    .allowAliasToMultipleIndices(false)
+                    .allowClosedIndices(true)
+                    .ignoreThrottled(false)
+                    .allowFailureIndices(false)
+                    .build()
+            )
+            .build();
 
         public Request(StreamInput in) throws IOException {
             super(in);

muted-tests.yml

@@ -71,9 +71,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=rollup/security_tests/Index-based access}
   issue: https://github.com/elastic/elasticsearch/issues/111631
-- class: org.elasticsearch.tdigest.ComparisonTests
-  method: testSparseGaussianDistribution
-  issue: https://github.com/elastic/elasticsearch/issues/111721
 - class: org.elasticsearch.upgrades.FullClusterRestartIT
   method: testSnapshotRestore {cluster=OLD}
   issue: https://github.com/elastic/elasticsearch/issues/111777
@@ -343,9 +340,6 @@ tests:
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=cluster.stats/30_ccs_stats/cross-cluster search stats search}
   issue: https://github.com/elastic/elasticsearch/issues/114371
-- class: org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT
-  method: testProfileOrdinalsGroupingOperator {SYNC}
-  issue: https://github.com/elastic/elasticsearch/issues/114380
 - class: org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests
   method: testInfer_StreamRequest
   issue: https://github.com/elastic/elasticsearch/issues/114385
@@ -384,6 +378,15 @@ tests:
 - class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT
   method: testTermsQuery
   issue: https://github.com/elastic/elasticsearch/issues/114563
+- class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT
+  method: testMatchAllQuery
+  issue: https://github.com/elastic/elasticsearch/issues/114607
+- class: org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizerTests
+  method: testPushSpatialIntersectsEvalToSource {default}
+  issue: https://github.com/elastic/elasticsearch/issues/114627
+- class: org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizerTests
+  method: testPushWhereEvalToSource {default}
+  issue: https://github.com/elastic/elasticsearch/issues/114628
 
 # Examples:
 #

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/70_dense_vector_telemetry.yml

@@ -21,13 +21,13 @@ setup:
                 element_type: byte
                 index_options:
                   type: hnsw
+                  m: 16
+                  ef_construction: 100
               vector2:
                 type: dense_vector
                 dims: 1024
                 index: true
                 similarity: dot_product
-                index_options:
-                  type: int8_hnsw
               vector3:
                 type: dense_vector
                 dims: 100

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -239,6 +239,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion TEXT_SIMILARITY_RERANKER_QUERY_REWRITE = def(8_763_00_0);
     public static final TransportVersion SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS = def(8_764_00_0);
     public static final TransportVersion RETRIEVERS_TELEMETRY_ADDED = def(8_765_00_0);
+    public static final TransportVersion ESQL_CACHED_STRING_SERIALIZATION = def(8_766_00_0);
+    public static final TransportVersion CHUNK_SENTENCE_OVERLAP_SETTING_ADDED = def(8_767_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesRequest.java

@@ -58,7 +58,26 @@ public class IndicesAliasesRequest extends AcknowledgedRequest<IndicesAliasesReq
     // indices options that require every specified index to exist, expand wildcards only to open
     // indices, don't allow that no indices are resolved from wildcard expressions and resolve the
     // expressions only against indices
-    private static final IndicesOptions INDICES_OPTIONS = IndicesOptions.fromOptions(false, false, true, false, true, false, true, false);
+    private static final IndicesOptions INDICES_OPTIONS = IndicesOptions.builder()
+        .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+        .wildcardOptions(
+            IndicesOptions.WildcardOptions.builder()
+                .matchOpen(true)
+                .matchClosed(false)
+                .includeHidden(false)
+                .resolveAliases(false)
+                .allowEmptyExpressions(false)
+                .build()
+        )
+        .gatekeeperOptions(
+            IndicesOptions.GatekeeperOptions.builder()
+                .allowAliasToMultipleIndices(true)
+                .allowClosedIndices(true)
+                .ignoreThrottled(false)
+                .allowFailureIndices(true)
+                .build()
+        )
+        .build();
 
     public IndicesAliasesRequest(StreamInput in) throws IOException {
         super(in);

server/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java

@@ -28,16 +28,25 @@
  */
 public class DeleteIndexRequest extends AcknowledgedRequest<DeleteIndexRequest> implements IndicesRequest.Replaceable {
 
-    public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.fromOptions(
-        false,
-        true,
-        true,
-        true,
-        false,
-        false,
-        true,
-        false
-    );
+    public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.builder()
+        .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+        .wildcardOptions(
+            IndicesOptions.WildcardOptions.builder()
+                .matchOpen(true)
+                .matchClosed(true)
+                .allowEmptyExpressions(true)
+                .resolveAliases(false)
+                .build()
+        )
+        .gatekeeperOptions(
+            IndicesOptions.GatekeeperOptions.builder()
+                .allowAliasToMultipleIndices(false)
+                .allowClosedIndices(true)
+                .ignoreThrottled(false)
+                .allowFailureIndices(true)
+                .build()
+        )
+        .build();
 
     private String[] indices;
     // Delete index should work by default on both open and closed indices.

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java

@@ -280,6 +280,7 @@ public static Template resolveTemplate(
                 templateSettings,
                 mappings
             );
+            MetadataCreateIndexService.validateAdditionalSettings(provider, result, additionalSettings);
             dummySettings.put(result);
             additionalSettings.put(result);
         }

server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java

@@ -40,7 +40,32 @@ public static class Request extends BroadcastRequest<Request> {
         public Request() {
             // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and
             // a data stream's backing indices are retrieved from its metadata
-            super(null, IndicesOptions.fromOptions(false, true, true, true, true, false, true, false));
+            super(
+                null,
+                IndicesOptions.builder()
+                    .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+                    .wildcardOptions(
+                        IndicesOptions.WildcardOptions.builder()
+                            .matchOpen(true)
+                            .matchClosed(true)
+                            .includeHidden(false)
+                            .resolveAliases(false)
+                            .allowEmptyExpressions(true)
+                            .build()
+                    )
+                    .gatekeeperOptions(
+                        IndicesOptions.GatekeeperOptions.builder()
+                            .allowAliasToMultipleIndices(true)
+                            .allowClosedIndices(true)
+                            .ignoreThrottled(false)
+                            .allowFailureIndices(true)
+                            .build()
+                    )
+                    .failureStoreOptions(
+                        IndicesOptions.FailureStoreOptions.builder().includeRegularIndices(true).includeFailureIndices(true).build()
+                    )
+                    .build()
+            );
         }
 
         public Request(StreamInput in) throws IOException {

server/src/main/java/org/elasticsearch/action/datastreams/DeleteDataStreamAction.java

@@ -46,7 +46,25 @@ public static class Request extends MasterNodeRequest<Request> implements Indice
         // empty response can be returned in case wildcards were used or
         // 404 status code returned in case no wildcard were used.
         private final boolean wildcardExpressionsOriginallySpecified;
-        private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false);
+        private IndicesOptions indicesOptions = IndicesOptions.builder()
+            .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+            .wildcardOptions(
+                IndicesOptions.WildcardOptions.builder()
+                    .matchOpen(true)
+                    .matchClosed(true)
+                    .resolveAliases(false)
+                    .allowEmptyExpressions(true)
+                    .build()
+            )
+            .gatekeeperOptions(
+                IndicesOptions.GatekeeperOptions.builder()
+                    .allowAliasToMultipleIndices(false)
+                    .allowClosedIndices(true)
+                    .ignoreThrottled(false)
+                    .allowFailureIndices(true)
+                    .build()
+            )
+            .build();
 
         public Request(TimeValue masterNodeTimeout, String... names) {
             super(masterNodeTimeout);

server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java

@@ -56,7 +56,26 @@ private GetDataStreamAction() {
     public static class Request extends MasterNodeReadRequest<Request> implements IndicesRequest.Replaceable {
 
         private String[] names;
-        private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false);
+        private IndicesOptions indicesOptions = IndicesOptions.builder()
+            .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+            .wildcardOptions(
+                IndicesOptions.WildcardOptions.builder()
+                    .matchOpen(true)
+                    .matchClosed(true)
+                    .includeHidden(false)
+                    .resolveAliases(false)
+                    .allowEmptyExpressions(true)
+                    .build()
+            )
+            .gatekeeperOptions(
+                IndicesOptions.GatekeeperOptions.builder()
+                    .allowAliasToMultipleIndices(false)
+                    .allowClosedIndices(true)
+                    .ignoreThrottled(false)
+                    .allowFailureIndices(true)
+                    .build()
+            )
+            .build();
         private boolean includeDefaults = false;
         private boolean verbose = false;
 

server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java

@@ -47,7 +47,26 @@ private GetDataStreamLifecycleAction() {/* no instances */}
     public static class Request extends MasterNodeReadRequest<Request> implements IndicesRequest.Replaceable {
 
         private String[] names;
-        private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false);
+        private IndicesOptions indicesOptions = IndicesOptions.builder()
+            .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+            .wildcardOptions(
+                IndicesOptions.WildcardOptions.builder()
+                    .matchOpen(true)
+                    .matchClosed(true)
+                    .includeHidden(false)
+                    .resolveAliases(false)
+                    .allowEmptyExpressions(true)
+                    .build()
+            )
+            .gatekeeperOptions(
+                IndicesOptions.GatekeeperOptions.builder()
+                    .allowAliasToMultipleIndices(false)
+                    .allowClosedIndices(true)
+                    .ignoreThrottled(false)
+                    .allowFailureIndices(true)
+                    .build()
+            )
+            .build();
         private boolean includeDefaults = false;
 
         public Request(TimeValue masterNodeTimeout, String[] names) {

server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java

@@ -78,7 +78,26 @@ public static Request parseRequest(XContentParser parser, Factory factory) {
         }
 
         private String[] names;
-        private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false);
+        private IndicesOptions indicesOptions = IndicesOptions.builder()
+            .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
+            .wildcardOptions(
+                IndicesOptions.WildcardOptions.builder()
+                    .matchOpen(true)
+                    .matchClosed(true)
+                    .includeHidden(false)
+                    .resolveAliases(false)
+                    .allowEmptyExpressions(true)
+                    .build()
+            )
+            .gatekeeperOptions(
+                IndicesOptions.GatekeeperOptions.builder()
+                    .allowAliasToMultipleIndices(false)
+                    .allowClosedIndices(true)
+                    .ignoreThrottled(false)
+                    .allowFailureIndices(false)
+                    .build()
+            )
+            .build();
         private final DataStreamLifecycle lifecycle;
 
         public Request(StreamInput in) throws IOException {

server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java

@@ -116,7 +116,8 @@ private void innerRun() throws Exception {
         // still use DFS_QUERY_THEN_FETCH, which does not perform the "query and fetch" optimization during the query phase.
         final boolean queryAndFetchOptimization = searchPhaseShardResults.length() == 1
             && context.getRequest().hasKnnSearch() == false
-            && reducedQueryPhase.queryPhaseRankCoordinatorContext() == null;
+            && reducedQueryPhase.queryPhaseRankCoordinatorContext() == null
+            && (context.getRequest().source() == null || context.getRequest().source().rankBuilder() == null);
         if (queryAndFetchOptimization) {
             assert assertConsistentWithQueryAndFetchOptimization();
             // query AND fetch optimization

server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java

@@ -462,8 +462,7 @@ private static SearchHits getHits(
                     : "not enough hits fetched. index [" + index + "] length: " + fetchResult.hits().getHits().length;
                 SearchHit searchHit = fetchResult.hits().getHits()[index];
                 searchHit.shard(fetchResult.getSearchShardTarget());
-                if (reducedQueryPhase.queryPhaseRankCoordinatorContext != null) {
-                    assert shardDoc instanceof RankDoc;
+                if (shardDoc instanceof RankDoc) {
                     searchHit.setRank(((RankDoc) shardDoc).rank);
                     searchHit.score(shardDoc.score);
                     long shardAndDoc = ShardDocSortField.encodeShardAndDoc(shardDoc.shardIndex, shardDoc.doc);
@@ -735,6 +734,12 @@ static int getTopDocsSize(SearchRequest request) {
             return DEFAULT_SIZE;
         }
         SearchSourceBuilder source = request.source();
+        if (source.rankBuilder() != null) {
+            // if we have a RankBuilder defined, it needs to have access to all the documents in order to rerank them
+            // so we override size here and keep all `rank_window_size` docs.
+            // Pagination is taking place later through RankFeaturePhaseRankCoordinatorContext#rankAndPaginate
+            return source.rankBuilder().rankWindowSize();
+        }
         return (source.size() == -1 ? DEFAULT_SIZE : source.size()) + (source.from() == -1 ? SearchService.DEFAULT_FROM : source.from());
     }