Merge branch 'main' into remove_unnecessary_wrappers

idegtiarenko · Oct 31, 2023 · 82988b2 · 82988b2
2 parents 891a6e3 + c7ddfef
commit 82988b2
Show file tree

Hide file tree

Showing 590 changed files with 9,004 additions and 3,676 deletions.
diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+third-party-tests.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+third-party-tests.yml
@@ -2,7 +2,8 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+periodic+third-party-tests
     display-name: "elastic / elasticsearch # %BRANCH% - third party tests"
-    description: "Testing of the Elasticsearch %BRANCH% branch against third-party service integrations.\n"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     project-type: multijob
     node: master
     vault: []

diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+thrid-party-tests-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+thrid-party-tests-trigger.yml
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -22,3 +22,6 @@ x-pack/plugin/core/src/main/resources/fleet-* @elastic/fleet
 
 # Kibana Security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @elastic/kibana-security
+
+# APM
+x-pack/plugin/apm-data @elastic/apm-server
diff --git a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
@@ -155,3 +155,8 @@ org.elasticsearch.cluster.ClusterStateTaskExecutor$TaskContext#success(java.util
 
 @defaultMessage ClusterState#compatibilityVersions are for internal use only. Use ClusterState#getMinVersions or a different version. See TransportVersion javadocs for more info.
 org.elasticsearch.cluster.ClusterState#compatibilityVersions()
+
+@defaultMessage ClusterFeatures#nodeFeatures is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
+org.elasticsearch.cluster.ClusterFeatures#nodeFeatures()
+@defaultMessage ClusterFeatures#clusterHasFeature is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
+org.elasticsearch.cluster.ClusterFeatures#clusterHasFeature(org.elasticsearch.features.NodeFeature)
diff --git a/docs/changelog/100899.yaml b/docs/changelog/100899.yaml
@@ -0,0 +1,5 @@
+pr: 100899
+summary: Add methods for adding generation listeners with primary term
+area: Store
+type: enhancement
+issues: []
diff --git a/docs/changelog/100974.yaml b/docs/changelog/100974.yaml
@@ -0,0 +1,5 @@
+pr: 100974
+summary: Create new cluster state API for querying features present on a cluster
+area: "Infra/Core"
+type: feature
+issues: []
diff --git a/docs/changelog/101093.yaml b/docs/changelog/101093.yaml
@@ -0,0 +1,6 @@
+pr: 101093
+summary: Make IPAddress writeable
+area: Infra/Scripting
+type: bug
+issues:
+ - 101082
diff --git a/docs/changelog/101202.yaml b/docs/changelog/101202.yaml
@@ -0,0 +1,5 @@
+pr: 101202
+summary: Optimize `MurmurHash3`
+area: "Ingest Node"
+type: enhancement
+issues: []
diff --git a/docs/changelog/101362.yaml b/docs/changelog/101362.yaml
@@ -0,0 +1,6 @@
+pr: 101362
+summary: "ESQL: Remove the swapped-args check for date_xxx()"
+area: ES|QL
+type: enhancement
+issues:
+ - 99562
diff --git a/docs/changelog/101385.yaml b/docs/changelog/101385.yaml
@@ -0,0 +1,6 @@
+pr: 101385
+summary: "ESQL: Fix planning of MV_EXPAND with foldable expressions"
+area: ES|QL
+type: bug
+issues:
+ - 101118
diff --git a/docs/changelog/101392.yaml b/docs/changelog/101392.yaml
@@ -0,0 +1,5 @@
+pr: 101392
+summary: Include ML processor limits in `_ml/info` response
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/101438.yaml b/docs/changelog/101438.yaml
@@ -0,0 +1,6 @@
+pr: 101438
+summary: "ESQL: Fix eval of functions on foldable literals"
+area: ES|QL
+type: bug
+issues:
+ - 101425
diff --git a/docs/changelog/101456.yaml b/docs/changelog/101456.yaml
@@ -0,0 +1,6 @@
+pr: 101456
+summary: "ESQL: adds Enrich implicit `match_fields` to `field_caps` call"
+area: ES|QL
+type: bug
+issues:
+ - 101328
diff --git a/docs/changelog/101457.yaml b/docs/changelog/101457.yaml
@@ -0,0 +1,14 @@
+pr: 101457
+summary: "Remove Plugin.createComponents method in favour of overload with a PluginServices object"
+area: Infra/Plugins
+type: breaking-java
+breaking:
+  area: "Java API"
+  title: "Plugin.createComponents method has been refactored to take a single PluginServices object"
+  details: >
+    Plugin.createComponents currently takes several different service arguments. The signature of this method changes
+    every time a new service is added. The method has now been modified to take a single interface object
+    that new services are added to. This will reduce API incompatibility issues when a new service
+    is introduced in the future.
+  impact: "Plugins that override createComponents will need to be refactored to override the new method on ES 8.12+"
+  notable: false
diff --git a/docs/changelog/101474.yaml b/docs/changelog/101474.yaml
@@ -0,0 +1,5 @@
+pr: 101474
+summary: "[Search Applications] Return 400 response when template rendering produces invalid JSON"
+area: Application
+type: bug
+issues: []
diff --git a/docs/changelog/101486.yaml b/docs/changelog/101486.yaml
@@ -0,0 +1,5 @@
+pr: 101486
+summary: Improving tika handling
+area: Ingest Node
+type: bug
+issues: []
diff --git a/docs/changelog/101488.yaml b/docs/changelog/101488.yaml
@@ -0,0 +1,5 @@
+pr: 101488
+summary: "ESQL: More tracking in `BlockHash` impls"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/101492.yaml b/docs/changelog/101492.yaml
@@ -0,0 +1,6 @@
+pr: 101492
+summary: "ESQL: check type before casting"
+area: ES|QL
+type: bug
+issues:
+ - 101489
diff --git a/docs/changelog/101495.yaml b/docs/changelog/101495.yaml
@@ -0,0 +1,5 @@
+pr: 101495
+summary: "[DSL] skip deleting indices that have in-progress downsampling operations"
+area: Data streams
+type: bug
+issues: []
diff --git a/docs/changelog/101497.yaml b/docs/changelog/101497.yaml
@@ -0,0 +1,5 @@
+pr: 101497
+summary: Fix snapshot double finalization
+area: Snapshot/Restore
+type: bug
+issues: []
diff --git a/docs/changelog/101518.yaml b/docs/changelog/101518.yaml
@@ -0,0 +1,6 @@
+pr: 101518
+summary: Check that scripts produce correct json in render template action
+area: Search
+type: bug
+issues:
+ - 101477
diff --git a/docs/changelog/101535.yaml b/docs/changelog/101535.yaml
@@ -0,0 +1,5 @@
+pr: 101535
+summary: Disable inter-segment concurrency when sorting by field
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/98244.yaml b/docs/changelog/98244.yaml
diff --git a/docs/changelog/99752.yaml b/docs/changelog/99752.yaml
@@ -0,0 +1,5 @@
+pr: 99752
+summary: Pass shard's primary term to Engine#addSegmentGenerationListener
+area: Store
+type: enhancement
+issues: []
diff --git a/docs/reference/esql/functions/signature/auto_bucket.svg b/docs/reference/esql/functions/signature/auto_bucket.svg
diff --git a/docs/reference/esql/functions/types/auto_bucket.asciidoc b/docs/reference/esql/functions/types/auto_bucket.asciidoc
@@ -1,5 +1,5 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
-arg1 | arg2 | arg3 | arg4 | result
+field | buckets | from | to | result
 
 |===
diff --git a/docs/reference/esql/metadata-fields.asciidoc b/docs/reference/esql/metadata-fields.asciidoc
@@ -34,11 +34,11 @@ like the other index fields:
 
 [source.merge.styled,esql]
 ----
-include::{esql-specs}/metadata-ignoreCsvTests.csv-spec[tag=multipleIndices]
+include::{esql-specs}/metadata-IT_tests_only.csv-spec[tag=multipleIndices]
 ----
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
-include::{esql-specs}/metadata-ignoreCsvTests.csv-spec[tag=multipleIndices-result]
+include::{esql-specs}/metadata-IT_tests_only.csv-spec[tag=multipleIndices-result]
 |===
 
 Also, similar to the index fields, once an aggregation is performed, a
@@ -47,9 +47,9 @@ used as grouping field:
 
 [source.merge.styled,esql]
 ----
-include::{esql-specs}/metadata-ignoreCsvTests.csv-spec[tag=metaIndexInAggs]
+include::{esql-specs}/metadata-IT_tests_only.csv-spec[tag=metaIndexInAggs]
 ----
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
-include::{esql-specs}/metadata-ignoreCsvTests.csv-spec[tag=metaIndexInAggs-result]
+include::{esql-specs}/metadata-IT_tests_only.csv-spec[tag=metaIndexInAggs-result]
 |===
diff --git a/docs/reference/esql/processing-commands/enrich.asciidoc b/docs/reference/esql/processing-commands/enrich.asciidoc
@@ -49,51 +49,51 @@ column for each enrich field defined in the policy. The match is performed using
 the `match_field` defined in the <<esql-enrich-policy,enrich policy>> and
 requires that the input table has a column with the same name (`language_code`
 in this example). `ENRICH` will look for records in the
-<<esql-enrich-index,enrich index>> based on the match field value. 
+<<esql-enrich-index,enrich index>> based on the match field value.
 
 [source.merge.styled,esql]
 ----
-include::{esql-specs}/docs-ignoreCsvTests.csv-spec[tag=enrich]
+include::{esql-specs}/docs-IT_tests_only.csv-spec[tag=enrich]
 ----
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
-include::{esql-specs}/docs-ignoreCsvTests.csv-spec[tag=enrich-result]
+include::{esql-specs}/docs-IT_tests_only.csv-spec[tag=enrich-result]
 |===
 
 To use a column with a different name than the `match_field` defined in the
 policy as the match field, use `ON <column-name>`:
 
 [source.merge.styled,esql]
 ----
-include::{esql-specs}/docs-ignoreCsvTests.csv-spec[tag=enrich_on]
+include::{esql-specs}/docs-IT_tests_only.csv-spec[tag=enrich_on]
 ----
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
-include::{esql-specs}/docs-ignoreCsvTests.csv-spec[tag=enrich_on-result]
+include::{esql-specs}/docs-IT_tests_only.csv-spec[tag=enrich_on-result]
 |===
 
 By default, each of the enrich fields defined in the policy is added as a
-column. To explicitly select the enrich fields that are added, use 
+column. To explicitly select the enrich fields that are added, use
 `WITH <field1>, <field2>...`:
 
 [source.merge.styled,esql]
 ----
-include::{esql-specs}/docs-ignoreCsvTests.csv-spec[tag=enrich_with]
+include::{esql-specs}/docs-IT_tests_only.csv-spec[tag=enrich_with]
 ----
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
-include::{esql-specs}/docs-ignoreCsvTests.csv-spec[tag=enrich_with-result]
+include::{esql-specs}/docs-IT_tests_only.csv-spec[tag=enrich_with-result]
 |===
 
 You can rename the columns that are added using `WITH new_name=<field1>`:
 
 [source.merge.styled,esql]
 ----
-include::{esql-specs}/docs-ignoreCsvTests.csv-spec[tag=enrich_rename]
+include::{esql-specs}/docs-IT_tests_only.csv-spec[tag=enrich_rename]
 ----
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
-include::{esql-specs}/docs-ignoreCsvTests.csv-spec[tag=enrich_rename-result]
+include::{esql-specs}/docs-IT_tests_only.csv-spec[tag=enrich_rename-result]
 |===
 
 In case of name collisions, the newly created columns will override existing

diff --git a/docs/reference/ml/common/apis/get-ml-info.asciidoc b/docs/reference/ml/common/apis/get-ml-info.asciidoc
@@ -120,7 +120,9 @@ This is a possible response:
   },
   "limits" : {
     "effective_max_model_memory_limit": "28961mb",
-    "total_ml_memory": "86883mb"
+    "total_ml_memory": "86883mb",
+    "total_ml_processors": 16,
+    "max_single_ml_node_processors": 16
   }
 }
 ----
@@ -129,3 +131,5 @@ This is a possible response:
 // TESTRESPONSE[s/"build_hash": "99a07c016d5a73"/"build_hash": "$body.native_code.build_hash"/]
 // TESTRESPONSE[s/"effective_max_model_memory_limit": "28961mb"/"effective_max_model_memory_limit": "$body.limits.effective_max_model_memory_limit"/]
 // TESTRESPONSE[s/"total_ml_memory": "86883mb"/"total_ml_memory": "$body.limits.total_ml_memory"/]
+// TESTRESPONSE[s/"total_ml_processors": 16/"total_ml_processors": $body.limits.total_ml_processors/]
+// TESTRESPONSE[s/"max_single_ml_node_processors": 16/"max_single_ml_node_processors": $body.limits.max_single_ml_node_processors/]
diff --git a/docs/reference/modules/discovery/bootstrapping.asciidoc b/docs/reference/modules/discovery/bootstrapping.asciidoc
@@ -151,7 +151,8 @@ a separate single-node cluster then you must start again:
 folder>>.
 
 . Configure `discovery.seed_hosts` or `discovery.seed_providers` and other
-relevant discovery settings.
+relevant discovery settings. Ensure `cluster.initial_master_nodes` is not set
+on any node.
 
 . Restart the node and verify that it joins the existing cluster rather than
 forming its own one-node cluster.
@@ -171,4 +172,6 @@ relevant discovery settings.
 
 . Restart all the nodes and verify that they have formed a single cluster.
 
+. Remove `cluster.initial_master_nodes` from every node's configuration.
+
 ****
diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc
@@ -149,6 +149,9 @@ Each query accepts a `_name` in its top level definition. You can use named
 queries to track which queries matched returned documents. If named queries are
 used, the response includes a `matched_queries` property for each hit.
 
+NOTE: Supplying duplicate `_name` values in the same request results in undefined behavior. Queries with duplicate names may overwrite
+each other. Query names are assumed to be unique within a single request.
+
 [source,console]
 ----
 GET /_search

diff --git a/docs/reference/query-dsl/fuzzy-query.asciidoc b/docs/reference/query-dsl/fuzzy-query.asciidoc
@@ -54,7 +54,7 @@ GET /_search
         "max_expansions": 50,
         "prefix_length": 0,
         "transpositions": true,
-        "rewrite": "constant_score"
+        "rewrite": "constant_score_blended"
       }
     }
   }

diff --git a/docs/reference/query-dsl/multi-term-rewrite.asciidoc b/docs/reference/query-dsl/multi-term-rewrite.asciidoc
@@ -29,7 +29,19 @@ query or bit set
 [[rewrite-param-valid-values]]
 === Valid values
 
-`constant_score` (Default)::
+`constant_score_blended` (Default)::
+Assigns each document a relevance score equal to the `boost`
+parameter.
++
+This method maintains a <<query-dsl-bool-query, `bool`
+query>> like implementation over the most costly terms while pre-processing
+the less costly terms into a filter bitset.
++
+This method can cause the generated `bool` query to exceed the clause limit in the
+<<indices-query-bool-max-clause-count, `indices.query.bool.max_clause_count`>>
+setting. If the query exceeds this limit, {es} returns an error.
+
+`constant_score` ::
 Uses the `constant_score_boolean` method for fewer matching terms. Otherwise,
 this method finds all matching terms in sequence and returns matching documents
 using a bit set.
@@ -102,8 +114,8 @@ setting.
 [discrete]
 [[rewrite-param-perf-considerations]]
 === Performance considerations for the `rewrite` parameter
-For most uses, we recommend using the `constant_score`,
+For most uses, we recommend using the  `constant_score_blended`, `constant_score`,
 `constant_score_boolean`, or `top_terms_boost_N` rewrite methods.
 
 Other methods calculate relevance scores. These score calculations are often
-expensive and do not improve query results.
+expensive and do not improve query results.
diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -685,3 +685,6 @@ a different index configuration, like the number of primary shards.
 ===== Allow expensive queries
 Percolate queries will not be executed if <<query-dsl-allow-expensive-queries, `search.allow_expensive_queries`>>
 is set to false.
+
+===== Using custom similarities
+Percolate queries will not respect any configured <<index-modules-similarity, custom similarity>>. They always use the default Lucene similarity.
diff --git a/docs/reference/query-dsl/regexp-query.asciidoc b/docs/reference/query-dsl/regexp-query.asciidoc
@@ -30,7 +30,7 @@ GET /_search
         "flags": "ALL",
         "case_insensitive": true,
         "max_determinized_states": 10000,
-        "rewrite": "constant_score"
+        "rewrite": "constant_score_blended"
       }
     }
   }

diff --git a/docs/reference/query-dsl/wildcard-query.asciidoc b/docs/reference/query-dsl/wildcard-query.asciidoc
@@ -26,7 +26,7 @@ GET /_search
       "user.id": {
         "value": "ki*y",
         "boost": 1.0,
-        "rewrite": "constant_score"
+        "rewrite": "constant_score_blended"
       }
     }
   }

diff --git a/docs/reference/search-application/apis/get-search-application.asciidoc b/docs/reference/search-application/apis/get-search-application.asciidoc
@@ -120,10 +120,3 @@ A sample response:
 }
 ----
 // TESTRESPONSE[s/"updated_at_millis": 1682105622204/"updated_at_millis": $body.$_path/]
-
-[NOTE]
-====
-The indices associated with a search application are not returned with the GET response.
-To view the indices, use the <<indices-get-alias, get alias>> API.
-The alias name will match the search application name, for example `my-app` in the example above.
-====