Merge branch 'validate_empty_index_names' of github.com:craigtaverner…

…/elasticsearch into validate_empty_index_names

docs/changelog/118968.yaml

@@ -0,0 +1,6 @@
+pr: 118968
+summary: Configure index sorting through index settings for logsdb
+area: Logs
+type: enhancement
+issues:
+ - 118686

docs/changelog/119730.yaml

@@ -0,0 +1,5 @@
+pr: 119730
+summary: Enable KQL function as a tech preview
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/119780.yaml

@@ -0,0 +1,5 @@
+pr: 119780
+summary: Add index and reindex request settings to speed up reindex
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/119863.yaml

@@ -0,0 +1,11 @@
+pr: 119863
+summary: Restrict Connector APIs to manage/monitor_connector privileges
+area: Extract&Transform
+type: breaking
+issues: []
+breaking:
+  title: Restrict Connector APIs to manage/monitor_connector privileges
+  area: REST API
+  details: Connector APIs now enforce the manage_connector and monitor_connector privileges (introduced in 8.15), replacing the previous reliance on index-level permissions for .elastic-connectors and .elastic-connectors-sync-jobs in API calls.
+  impact: Connector APIs now require manage_connector and monitor_connector privileges
+  notable: false

docs/reference/connector/docs/connectors-sharepoint-online.asciidoc

@@ -75,12 +75,10 @@ Follow these steps:
 * Leave the *Redirect URIs* blank for now.
 * *Register* the application.
 * Find and keep the **Application (client) ID** and **Directory (tenant) ID** handy.
-* Locate the **Secret** by navigating to **Client credentials: Certificates & Secrets**.
-* Select **New client secret**
-* Pick a name for your client secret.
-Select an expiration date. (At this expiration date, you will need to generate a new secret and update your connector configuration.)
-** Save the client secret **Secret ID** before leaving this screen.
-** Save the client secret **Value** before leaving this screen.
+* Create a certificate and private key. This can, for example, be done by running `openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout azure_app.key -out azure_app.crt` command. Store both in a safe and secure place
+* Locate the **Certificates** by navigating to **Client credentials: Certificates & Secrets**.
+* Select **Upload certificate**
+* Upload the certificate created in one of previous steps: `azure_app.crt`
 * Set up the permissions the OAuth App will request from the Azure Portal service account.
 ** Navigate to **API Permissions** and click **Add Permission**.
 ** Add **application permissions** until the list looks like the following:
@@ -114,6 +112,24 @@ When entities are not available via the Graph API the connector falls back to us
 [discrete#es-connectors-sharepoint-online-oauth-app-permissions]
 ====== SharePoint permissions
 
+Microsoft is https://learn.microsoft.com/en-us/sharepoint/dev/sp-add-ins/retirement-announcement-for-azure-acs[retiring Azure Access Control Service (ACS)]. This affects permission configuration:
+
+* *Tenants created after November 1st, 2024*: Certificate authentication is required
+* *Tenants created before November 1st, 2024*: Secret-based authentication must be migrated to certificate authentication by April 2nd, 2026
+
+[discrete#es-connectors-sharepoint-online-oauth-app-certificate-auth]
+===== Certificate Authentication
+
+This authentication method does not require additional setup other than creating and uploading certificates to the OAuth App.
+
+[discrete#es-connectors-sharepoint-online-oauth-app-secret-auth]
+===== Secret Authentication
+
+[IMPORTANT]
+====
+This method is only applicable to tenants created before November 1st, 2024. This method will be fully retired as of April 2nd, 2026.
+====
+
 Refer to the following documentation for setting https://learn.microsoft.com/en-us/sharepoint/dev/solution-guidance/security-apponly-azureacs[SharePoint permissions^].
 
 * To set `DisableCustomAppAuthentication` to false, connect to SharePoint using PowerShell and run `set-spotenant -DisableCustomAppAuthentication $false`
@@ -219,8 +235,17 @@ The tenant name for the Azure account hosting the Sharepoint Online instance.
 Client ID::
 The client id to authenticate with SharePoint Online.
 
+Authentication Method::
+Authentication method to use to connector to Sharepoint Online and Rest APIs. `secret` is deprecated and `certificate` is recommended.
+
 Secret value::
-The secret value to authenticate with SharePoint Online.
+The secret value to authenticate with SharePoint Online, if Authentication Method: `secret` is chosen.
+
+Content of certificate file::
+Content of certificate file if Authentication Method: `certificate` is chosen.
+
+Content of private key file::
+Content of private key file if Authentication Method: `certificate` is chosen.
 
 Comma-separated list of sites::
 List of site collection names or paths to fetch from SharePoint.
@@ -588,12 +613,10 @@ Follow these steps:
 * Leave the *Redirect URIs* blank for now.
 * *Register* the application.
 * Find and keep the **Application (client) ID** and **Directory (tenant) ID** handy.
-* Locate the **Secret** by navigating to **Client credentials: Certificates & Secrets**.
-* Select **New client secret**
-* Pick a name for your client secret.
-Select an expiration date. (At this expiration date, you will need to generate a new secret and update your connector configuration.)
-** Save the client secret **Secret ID** before leaving this screen.
-** Save the client secret **Value** before leaving this screen.
+* Create a certificate and private key. This can, for example, be done by running `openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout azure_app.key -out azure_app.crt` command. Store both in a safe and secure place
+* Locate the **Certificates** by navigating to **Client credentials: Certificates & Secrets**.
+* Select **Upload certificate**
+* Upload the certificate created in one of previous steps: `azure_app.crt`
 * Set up the permissions the OAuth App will request from the Azure Portal service account.
 ** Navigate to **API Permissions** and click **Add Permission**.
 ** Add **application permissions** until the list looks like the following:
@@ -627,6 +650,23 @@ When entities are not available via the Graph API the connector falls back to us
 [discrete#es-connectors-sharepoint-online-client-oauth-app-permissions]
 ====== SharePoint permissions
 
+Microsoft is https://learn.microsoft.com/en-us/sharepoint/dev/sp-add-ins/retirement-announcement-for-azure-acs[retiring Azure Access Control Service (ACS)]. This affects permission configuration:
+* *Tenants created after November 1st, 2024*: Certificate authentication is required
+* *Tenants created before November 1st, 2024*: Secret-based authentication must be migrated to certificate authentication by April 2nd, 2026
+
+[discrete#es-connectors-sharepoint-online-client-oauth-app-certificate-auth]
+===== Certificate Authentication
+
+This authentication method does not require additional setup other than creating and uploading certificates to the OAuth App.
+
+[discrete#es-connectors-sharepoint-online-client-oauth-app-secret-auth]
+===== Secret Authentication
+
+[IMPORTANT]
+====
+This method is only applicable to tenants created before November 1st, 2024. This method will be fully retired as of April 2nd, 2026.
+====
+
 Refer to the following documentation for setting https://learn.microsoft.com/en-us/sharepoint/dev/solution-guidance/security-apponly-azureacs[SharePoint permissions^].
 
 * To set `DisableCustomAppAuthentication` to false, connect to SharePoint using PowerShell and run `set-spotenant -DisableCustomAppAuthentication $false`
@@ -742,8 +782,17 @@ The tenant name for the Azure account hosting the Sharepoint Online instance.
 `client_id`::
 The client id to authenticate with SharePoint Online.
 
+`auth_method`::
+Authentication method to use to connector to Sharepoint Online and Rest APIs. `secret` is deprecated and `certificate` is recommended.
+
 `secret_value`::
-The secret value to authenticate with SharePoint Online.
+The secret value to authenticate with SharePoint Online, if auth_method: `secret` is chosen.
+
+`certificate`::
+Content of certificate file if auth_method: `certificate` is chosen.
+
+`private_key`::
+Content of private key file if auth_method: `certificate` is chosen.
 
 `site_collections`::
 List of site collection names or paths to fetch from SharePoint.

docs/reference/esql/esql-limitations.asciidoc

@@ -150,7 +150,7 @@ FROM books
 
 Note that, because of <<esql-limitations-text-fields,the way {esql} treats `text` values>>,
 any queries on `text` fields that do not explicitly use the full-text functions,
-<<esql-match>> or <<esql-qstr>>, will behave as if the fields are actually `keyword` fields:
+<<esql-match>>, <<esql-qstr>> or <<esql-kql>>, will behave as if the fields are actually `keyword` fields:
 they are case-sensitive and need to match the full string.
 
 [discrete]

docs/reference/esql/functions/search-functions.asciidoc

@@ -18,9 +18,11 @@ See <<esql-limitations-full-text-search,full text search limitations>> for infor
 {esql} supports these full-text search functions:
 
 // tag::search_list[]
+* experimental:[] <<esql-kql>>
 * experimental:[] <<esql-match>>
 * experimental:[] <<esql-qstr>>
 // end::search_list[]
 
+include::layout/kql.asciidoc[]
 include::layout/match.asciidoc[]
 include::layout/qstr.asciidoc[]

modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/SearchCancellationIT.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.aggregations.AggregationsPlugin;
 import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder;
+import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
@@ -97,9 +98,11 @@ public void testCancellationDuringTimeSeriesAggregation() throws Exception {
 
         logger.info("Executing search");
         // we have to explicitly set error_trace=true for the later exception check for `TimeSeriesIndexSearcher`
-        client().threadPool().getThreadContext().putHeader("error_trace", "true");
+        Client client = client();
+        client.threadPool().getThreadContext().putHeader("error_trace", "true");
         TimeSeriesAggregationBuilder timeSeriesAggregationBuilder = new TimeSeriesAggregationBuilder("test_agg");
-        ActionFuture<SearchResponse> searchResponse = prepareSearch("test").setQuery(matchAllQuery())
+        ActionFuture<SearchResponse> searchResponse = client.prepareSearch("test")
+            .setQuery(matchAllQuery())
             .addAggregation(
                 timeSeriesAggregationBuilder.subAggregation(
                     new ScriptedMetricAggregationBuilder("sub_agg").initScript(

muted-tests.yml

@@ -155,12 +155,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.RegressionIT
   method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet
   issue: https://github.com/elastic/elasticsearch/issues/117805
-- class: org.elasticsearch.upgrades.QueryBuilderBWCIT
-  method: testQueryBuilderBWC {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/116990
-- class: org.elasticsearch.xpack.restart.QueryBuilderBWCIT
-  method: testQueryBuilderBWC {p0=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/116989
 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT
   method: testEsqlRcs2UnavailableRemoteScenarios
   issue: https://github.com/elastic/elasticsearch/issues/117419
@@ -202,9 +196,6 @@ tests:
 - class: org.elasticsearch.cluster.service.MasterServiceTests
   method: testThreadContext
   issue: https://github.com/elastic/elasticsearch/issues/118914
-- class: org.elasticsearch.aggregations.bucket.SearchCancellationIT
-  method: testCancellationDuringTimeSeriesAggregation
-  issue: https://github.com/elastic/elasticsearch/issues/118992
 - class: org.elasticsearch.xpack.security.authc.AuthenticationServiceTests
   method: testInvalidToken
   issue: https://github.com/elastic/elasticsearch/issues/119019
@@ -254,8 +245,9 @@ tests:
 - class: org.elasticsearch.xpack.security.authz.IndicesAndAliasesResolverTests
   method: testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildcard
   issue: https://github.com/elastic/elasticsearch/issues/119909
-- class: org.elasticsearch.xpack.esql.CsvTests
-  issue: https://github.com/elastic/elasticsearch/issues/119918
+- class: org.elasticsearch.xpack.inference.InferenceCrudIT
+  method: testGetServicesWithCompletionTaskType
+  issue: https://github.com/elastic/elasticsearch/issues/119959
 
 # Examples:
 #

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java

@@ -12,6 +12,8 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 
 import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
@@ -21,6 +23,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.DisMaxQueryBuilder;
@@ -51,6 +54,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.cluster.ClusterState.VERSION_INTRODUCING_TRANSPORT_VERSIONS;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 
 /**
@@ -245,7 +249,19 @@ public void testQueryBuilderBWC() throws Exception {
                     InputStream in = new ByteArrayInputStream(qbSource, 0, qbSource.length);
                     StreamInput input = new NamedWriteableAwareStreamInput(new InputStreamStreamInput(in), registry)
                 ) {
-                    input.setTransportVersion(TransportVersion.readVersion(input));
+                    @UpdateForV10(owner = UpdateForV10.Owner.SEARCH_FOUNDATIONS)    // won't need to read <8.8 data anymore
+                    boolean originalClusterHasTransportVersion = parseLegacyVersion(getOldClusterVersion()).map(
+                        v -> v.onOrAfter(VERSION_INTRODUCING_TRANSPORT_VERSIONS)
+                    ).orElse(true);
+                    TransportVersion transportVersion;
+                    if (originalClusterHasTransportVersion == false) {
+                        transportVersion = TransportVersion.fromId(
+                            parseLegacyVersion(getOldClusterVersion()).map(Version::id).orElse(TransportVersions.MINIMUM_COMPATIBLE.id())
+                        );
+                    } else {
+                        transportVersion = TransportVersion.readVersion(input);
+                    }
+                    input.setTransportVersion(transportVersion);
                     QueryBuilder queryBuilder = input.readNamedWriteable(QueryBuilder.class);
                     assert in.read() == -1;
                     assertEquals(expectedQueryBuilder, queryBuilder);

rest-api-spec/build.gradle

@@ -60,6 +60,12 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
   task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0")
   task.skipTest("search/500_date_range/from, to, include_lower, include_upper deprecated", "deprecated parameters are removed in 9.0")
+  task.skipTest("logsdb/10_settings/logsdb with default ignore dynamic beyond limit and default sorting", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/logsdb with default ignore dynamic beyond limit and too low limit", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/logsdb with default ignore dynamic beyond limit and subobjects false", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/override sort missing settings", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/override sort order settings", "skip until pr/118968 gets backported")
+  task.skipTest("logsdb/10_settings/override sort mode settings", "skip until pr/118968 gets backported")
   task.skipTest("search.vectors/41_knn_search_bbq_hnsw/Test knn search", "Scoring has changed in latest versions")
   task.skipTest("search.vectors/42_knn_search_bbq_flat/Test knn search", "Scoring has changed in latest versions")
   task.skipTest("search.vectors/180_update_dense_vector_type/Test create and update dense vector mapping with bulk indexing", "waiting for #118774 backport")