Merge branch 'main' into st_extent_optimize_geoshape

docs/changelog/118968.yaml

@@ -0,0 +1,6 @@
+pr: 118968
+summary: Configure index sorting through index settings for logsdb
+area: Logs
+type: enhancement
+issues:
+ - 118686

docs/changelog/119730.yaml

@@ -0,0 +1,5 @@
+pr: 119730
+summary: Enable KQL function as a tech preview
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/119863.yaml

@@ -0,0 +1,11 @@
+pr: 119863
+summary: Restrict Connector APIs to manage/monitor_connector privileges
+area: Extract&Transform
+type: breaking
+issues: []
+breaking:
+  title: Restrict Connector APIs to manage/monitor_connector privileges
+  area: REST API
+  details: Connector APIs now enforce the manage_connector and monitor_connector privileges (introduced in 8.15), replacing the previous reliance on index-level permissions for .elastic-connectors and .elastic-connectors-sync-jobs in API calls.
+  impact: Connector APIs now require manage_connector and monitor_connector privileges
+  notable: false

docs/reference/esql/esql-limitations.asciidoc

@@ -150,7 +150,7 @@ FROM books
 
 Note that, because of <<esql-limitations-text-fields,the way {esql} treats `text` values>>,
 any queries on `text` fields that do not explicitly use the full-text functions,
-<<esql-match>> or <<esql-qstr>>, will behave as if the fields are actually `keyword` fields:
+<<esql-match>>, <<esql-qstr>> or <<esql-kql>>, will behave as if the fields are actually `keyword` fields:
 they are case-sensitive and need to match the full string.
 
 [discrete]

docs/reference/esql/functions/search-functions.asciidoc

@@ -18,9 +18,11 @@ See <<esql-limitations-full-text-search,full text search limitations>> for infor
 {esql} supports these full-text search functions:
 
 // tag::search_list[]
+* experimental:[] <<esql-kql>>
 * experimental:[] <<esql-match>>
 * experimental:[] <<esql-qstr>>
 // end::search_list[]
 
+include::layout/kql.asciidoc[]
 include::layout/match.asciidoc[]
 include::layout/qstr.asciidoc[]

modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.aggregations.AggregationsPlugin;
 import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder;
+import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
@@ -97,9 +98,11 @@ public void testCancellationDuringTimeSeriesAggregation() throws Exception {
 
         logger.info("Executing search");
         // we have to explicitly set error_trace=true for the later exception check for `TimeSeriesIndexSearcher`
-        client().threadPool().getThreadContext().putHeader("error_trace", "true");
+        Client client = client();
+        client.threadPool().getThreadContext().putHeader("error_trace", "true");
         TimeSeriesAggregationBuilder timeSeriesAggregationBuilder = new TimeSeriesAggregationBuilder("test_agg");
-        ActionFuture<SearchResponse> searchResponse = prepareSearch("test").setQuery(matchAllQuery())
+        ActionFuture<SearchResponse> searchResponse = client.prepareSearch("test")
+            .setQuery(matchAllQuery())
             .addAggregation(
                 timeSeriesAggregationBuilder.subAggregation(
                     new ScriptedMetricAggregationBuilder("sub_agg").initScript(

muted-tests.yml

@@ -155,12 +155,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.RegressionIT
   method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet
   issue: https://github.com/elastic/elasticsearch/issues/117805
-- class: org.elasticsearch.upgrades.QueryBuilderBWCIT
-  method: testQueryBuilderBWC {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/116990
-- class: org.elasticsearch.xpack.restart.QueryBuilderBWCIT
-  method: testQueryBuilderBWC {p0=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/116989
 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT
   method: testEsqlRcs2UnavailableRemoteScenarios
   issue: https://github.com/elastic/elasticsearch/issues/117419
@@ -202,9 +196,6 @@ tests:
 - class: org.elasticsearch.cluster.service.MasterServiceTests
   method: testThreadContext
   issue: https://github.com/elastic/elasticsearch/issues/118914
-- class: org.elasticsearch.aggregations.bucket.SearchCancellationIT
-  method: testCancellationDuringTimeSeriesAggregation
-  issue: https://github.com/elastic/elasticsearch/issues/118992
 - class: org.elasticsearch.xpack.security.authc.AuthenticationServiceTests
   method: testInvalidToken
   issue: https://github.com/elastic/elasticsearch/issues/119019
@@ -254,8 +245,6 @@ tests:
 - class: org.elasticsearch.xpack.security.authz.IndicesAndAliasesResolverTests
   method: testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildcard
   issue: https://github.com/elastic/elasticsearch/issues/119909
-- class: org.elasticsearch.xpack.esql.CsvTests
-  issue: https://github.com/elastic/elasticsearch/issues/119918
 
 # Examples:
 #

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java

@@ -12,6 +12,8 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
@@ -21,6 +23,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.DisMaxQueryBuilder;
@@ -51,6 +54,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.cluster.ClusterState.VERSION_INTRODUCING_TRANSPORT_VERSIONS;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 
 /**
@@ -245,7 +249,19 @@ public void testQueryBuilderBWC() throws Exception {
                     InputStream in = new ByteArrayInputStream(qbSource, 0, qbSource.length);
                     StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in), registry)
                 ) {
-                    input.setTransportVersion(TransportVersion.readVersion(input));
+                    @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS)    // won't need to read <8.8 data anymore
+                    boolean originalClusterHasTransportVersion = parseLegacyVersion(getOldClusterVersion()).map(
+                        v -> v.onOrAfter(VERSION_INTRODUCING_TRANSPORT_VERSIONS)
+                    ).orElse(true);
+                    TransportVersion transportVersion;
+                    if (originalClusterHasTransportVersion == false) {
+                        transportVersion = TransportVersion.fromId(
+                            parseLegacyVersion(getOldClusterVersion()).map(Version::id).orElse(TransportVersions.MINIMUM_COMPATIBLE.id())
+                        );
+                    } else {
+                        transportVersion = TransportVersion.readVersion(input);
+                    }
+                    input.setTransportVersion(transportVersion);
                     QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
                     assert in.read() == -1;
                     assertEquals(expectedQueryBuilder, queryBuilder);

rest-api-spec/build.gradle

@@ -60,6 +60,12 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
   task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0")
   task.skipTest("search/500_date_range/from, to, include_lower, include_upper deprecated", "deprecated parameters are removed in 9.0")
+  task.skipTest("logsdb/10_settings/logsdb with default ignore dynamic beyond limit and default sorting", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/logsdb with default ignore dynamic beyond limit and too low limit", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/logsdb with default ignore dynamic beyond limit and subobjects false", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/override sort missing settings", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/override sort order settings", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/override sort mode settings", "skip until pr/118968 gets backported")
   task.skipTest("search.vectors/41_knn_search_bbq_hnsw/Test knn search", "Scoring has changed in latest versions")
   task.skipTest("search.vectors/42_knn_search_bbq_flat/Test knn search", "Scoring has changed in latest versions")
   task.skipTest("search.vectors/180_update_dense_vector_type/Test create and update dense vector mapping with bulk indexing", "waiting for #118774 backport")