Merge branch 'main' into feature/esql-match-operator-non-snapshot

# Conflicts:
#	x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens
#	x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -145,7 +145,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
 
     doLast {
       ['main', 'test'].each { sourceSet ->
-        modifyXml(".idea/modules/libs/native/elasticsearch.libs.${project.project(':libs:native').name}.${sourceSet}.iml") { xml ->
+        modifyXml(".idea/modules/libs/native/elasticsearch.libs.native.${sourceSet}.iml") { xml ->
           xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = 'JDK_21_PREVIEW'
         }
       }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java

@@ -137,7 +137,7 @@ public void skipTest(String fullTestName, String reason) {
         // However, the folder can be arbitrarily nest so, a == a1/a2/a3, and the test name can include forward slashes, so c == c1/c2/c3
         // So we also need to support a1/a2/a3/b/c1/c2/c3
 
-        String[] testParts = fullTestName.split("/");
+        String[] testParts = fullTestName.split("/", 3);
         if (testParts.length < 3) {
             throw new IllegalArgumentException(
                 "To skip tests, all 3 parts [folder/file/test name] must be defined. found [" + fullTestName + "]"

docs/changelog/115091.yaml

@@ -0,0 +1,7 @@
+pr: 115091
+summary: Added stricter range type checks and runtime warnings for ENRICH
+area: ES|QL
+type: bug
+issues:
+ - 107357
+ - 116799

docs/changelog/116515.yaml

@@ -0,0 +1,5 @@
+pr: 116515
+summary: Esql/lookup join grammar
+area: ES|QL
+type: feature
+issues: []

docs/changelog/116689.yaml

@@ -0,0 +1,10 @@
+pr: 116689
+summary: Deprecate `_source.mode` in mappings
+area: Mapping
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate `_source.mode` in mappings
+  area: Mapping
+  details: Configuring `_source.mode` in mappings is deprecated and will be removed in future versions. Use `index.mapping.source.mode` index setting instead.
+  impact: Use `index.mapping.source.mode` index setting instead

docs/changelog/116944.yaml

@@ -0,0 +1,11 @@
+pr: 116944
+summary: "Remove support for type, fields, `copy_to` and boost in metadata field definition"
+area: Mapping
+type: breaking
+issues: []
+breaking:
+  title: "Remove support for type, fields, copy_to and boost in metadata field definition"
+  area: Mapping
+  details: The type, fields, copy_to and boost parameters are no longer supported in metadata field definition
+  impact: Users providing type, fields, copy_to or boost as part of metadata field definition should remove them from their mappings.
+  notable: false

docs/changelog/116970.yaml

@@ -0,0 +1,11 @@
+pr: 116970
+summary: Remove legacy params from range query
+area: Search
+type: breaking
+issues: []
+breaking:
+  title: Remove legacy params from range query
+  area: REST API
+  details: The deprecated range query parameters `to`, `from`, `include_lower`, and `include_upper` are no longer supported.
+  impact: Users should use `lt`, `lte`, `gt`, and `gte` query parameters instead.
+  notable: false

docs/changelog/90529.yaml

@@ -0,0 +1,26 @@
+pr: 90529
+summary: Output a consistent format when generating error json
+area: Infra/REST API
+type: "breaking"
+issues:
+ - 89387
+breaking:
+  title: Error JSON structure has changed when detailed errors are disabled
+  area: REST API
+  details: |-
+    This change modifies the JSON format of error messages returned to REST clients
+    when detailed messages are turned off.
+    Previously, JSON returned when an exception occurred, and `http.detailed_errors.enabled: false` was set,
+    just consisted of a single `"error"` text field with some basic information.
+    Setting `http.detailed_errors.enabled: true` (the default) changed this field
+    to an object with more detailed information.
+    With this change, non-detailed errors now have the same structure as detailed errors. `"error"` will now always
+    be an object with, at a minimum, a `"type"` and `"reason"` field. Additional fields are included when detailed
+    errors are enabled.
+    To use the previous structure for non-detailed errors, use the v8 REST API.
+  impact: |-
+    If you have set `http.detailed_errors.enabled: false` (the default is `true`)
+    the structure of JSON when any exceptions occur now matches the structure when
+    detailed errors are enabled.
+    To use the previous structure for non-detailed errors, use the v8 REST API.
+  notable: false

docs/reference/cluster/allocation-explain.asciidoc

@@ -159,6 +159,8 @@ node.
 <5> The decider which led to the `no` decision for the node.
 <6> An explanation as to why the decider returned a `no` decision, with a helpful hint pointing to the setting that led to the decision. In this example, a newly created index has <<indices-get-settings,an index setting>> that requires that it only be allocated to a node named `nonexistent_node`, which does not exist, so the index is unable to allocate.
 
+See https://www.youtube.com/watch?v=5z3n2VgusLE[this video] for a walkthrough of troubleshooting a node and index setting mismatch.
+
 [[maximum-number-of-retries-exceeded]]
 ====== Maximum number of retries exceeded
 
@@ -235,7 +237,9 @@ primary shard that was previously allocated.
 ----
 // NOTCONSOLE
 
-TIP: If a shard is unassigned with an allocation status of `no_valid_shard_copy`, then you should <<fix-cluster-status-recover-nodes,make sure that all nodes are in the cluster>>. If all the nodes containing in-sync copies of a shard are lost, then you can <<fix-cluster-status-restore,recover the data for the shard>>.
+If a shard is unassigned with an allocation status of `no_valid_shard_copy`, then you should <<fix-cluster-status-recover-nodes,make sure that all nodes are in the cluster>>. If all the nodes containing in-sync copies of a shard are lost, then you can <<fix-cluster-status-restore,recover the data for the shard>>. 
+
+See https://www.youtube.com/watch?v=6OAg9IyXFO4[this video] for a walkthrough of troubleshooting `no_valid_shard_copy`.
 
 ===== Unassigned replica shard
 

docs/reference/esql/esql-commands.asciidoc

@@ -38,12 +38,12 @@ image::images/esql/processing-command.svg[A processing command changing an input
 * <<esql-eval>>
 * <<esql-grok>>
 ifeval::["{release-state}"=="unreleased"]
-* experimental:[] <<esql-inlinestats-by>>
+//* experimental:[] <<esql-inlinestats-by>>
 endif::[]
 * <<esql-keep>>
 * <<esql-limit>>
 ifeval::["{release-state}"=="unreleased"]
-* experimental:[] <<esql-lookup>>
+//* experimental:[] <<esql-lookup>>
 endif::[]
 * experimental:[] <<esql-mv_expand>>
 * <<esql-rename>>
@@ -63,12 +63,12 @@ include::processing-commands/enrich.asciidoc[]
 include::processing-commands/eval.asciidoc[]
 include::processing-commands/grok.asciidoc[]
 ifeval::["{release-state}"=="unreleased"]
-include::processing-commands/inlinestats.asciidoc[]
+//include::processing-commands/inlinestats.asciidoc[]
 endif::[]
 include::processing-commands/keep.asciidoc[]
 include::processing-commands/limit.asciidoc[]
 ifeval::["{release-state}"=="unreleased"]
-include::processing-commands/lookup.asciidoc[]
+//include::processing-commands/lookup.asciidoc[]
 endif::[]
 include::processing-commands/mv_expand.asciidoc[]
 include::processing-commands/rename.asciidoc[]

docs/reference/esql/esql-enrich-data.asciidoc

@@ -138,8 +138,33 @@ include::{es-ref-dir}/ingest/apis/enrich/execute-enrich-policy.asciidoc[tag=upda
 
 include::../ingest/enrich.asciidoc[tag=update-enrich-policy]
 
-==== Limitations
+==== Enrich Policy Types and Limitations
+The {esql} `ENRICH` command supports all three enrich policy types:
+
+`geo_match`::
+Matches enrich data to incoming documents based on a <<query-dsl-geo-shape-query,`geo_shape` query>>.
+For an example, see <<geo-match-enrich-policy-type>>.
+
+`match`::
+Matches enrich data to incoming documents based on a <<query-dsl-term-query,`term` query>>.
+For an example, see <<match-enrich-policy-type>>.
+
+`range`::
+Matches a number, date, or IP address in incoming documents to a range in the
+enrich index based on a <<query-dsl-term-query,`term` query>>. For an example,
+see <<range-enrich-policy-type>>.
+
 // tag::limitations[]
-The {esql} `ENRICH` command only supports enrich policies of type `match`.
-Furthermore, `ENRICH` only supports enriching on a column of type `keyword`.
+While all three enrich policy types are supported, there are some limitations to be aware of:
+
+* The `geo_match` enrich policy type only supports the `intersects` spatial relation.
+* It is required that the `match_field` in the `ENRICH` command is of the correct type.
+For example, if the enrich policy is of type `geo_match`, the `match_field` in the `ENRICH`
+command must be of type `geo_point` or `geo_shape`.
+Likewise, a `range` enrich policy requires a `match_field` of type `integer`, `long`, `date`, or `ip`,
+depending on the type of the range field in the original enrich index.
+* However, this constraint is relaxed for `range` policies when the `match_field` is of type `KEYWORD`.
+In this case the field values will be parsed during query execution, row by row.
+If any value fails to parse, the output values for that row will be set to `null`,
+an appropriate warning will be produced and the query will continue to execute.
 // end::limitations[]

docs/reference/esql/esql-language.asciidoc

@@ -14,6 +14,7 @@ Detailed reference documentation for the {esql} language:
 * <<esql-enrich-data>>
 * <<esql-process-data-with-dissect-and-grok>>
 * <<esql-implicit-casting>>
+* <<esql-time-spans>>
 
 include::esql-syntax.asciidoc[]
 include::esql-commands.asciidoc[]
@@ -23,3 +24,4 @@ include::multivalued-fields.asciidoc[]
 include::esql-process-data-with-dissect-grok.asciidoc[]
 include::esql-enrich-data.asciidoc[]
 include::implicit-casting.asciidoc[]
+include::time-spans.asciidoc[]

docs/reference/esql/esql-query-api.asciidoc

@@ -92,8 +92,8 @@ https://en.wikipedia.org/wiki/Query_plan[EXPLAIN PLAN].
 
 
 ifeval::["{release-state}"=="unreleased"]
-`table`::
-(Optional, object) Named "table" parameters that can be referenced by the <<esql-lookup>> command.
+//`table`::
+//(Optional, object) Named "table" parameters that can be referenced by the <<esql-lookup>> command.
 endif::[]
 
 [discrete]

docs/reference/esql/esql-syntax.asciidoc

@@ -157,21 +157,15 @@ FROM employees
 ==== Timespan literals
 
 Datetime intervals and timespans can be expressed using timespan literals.
-Timespan literals are a combination of a number and a qualifier. These
-qualifiers are supported:
-
-* `millisecond`/`milliseconds`/`ms`
-* `second`/`seconds`/`sec`/`s`
-* `minute`/`minutes`/`min`
-* `hour`/`hours`/`h`
-* `day`/`days`/`d`
-* `week`/`weeks`/`w`
-* `month`/`months`/`mo`
-* `quarter`/`quarters`/`q`
-* `year`/`years`/`yr`/`y`
+Timespan literals are a combination of a number and a temporal unit. The
+supported temporal units are listed in <<esql-time-spans-table, time span unit>>.
+More examples of the usages of time spans can be found in
+<<esql-time-spans, Use time spans in ES|QL>>.
+
 
 Timespan literals are not whitespace sensitive. These expressions are all valid:
 
 * `1day`
 * `1 day`
 * `1       day`
+

docs/reference/esql/functions/binary.asciidoc

@@ -87,6 +87,7 @@ Supported types:
 
 include::types/greater_than_or_equal.asciidoc[]
 
+[[esql-add]]
 ==== Add `+`
 [.text-center]
 image::esql/functions/signature/add.svg[Embedded,opts=inline]
@@ -98,6 +99,7 @@ Supported types:
 
 include::types/add.asciidoc[]
 
+[[esql-subtract]]
 ==== Subtract `-`
 [.text-center]
 image::esql/functions/signature/sub.svg[Embedded,opts=inline]

docs/reference/esql/implicit-casting.asciidoc

@@ -5,7 +5,7 @@
 <titleabbrev>Implicit casting</titleabbrev>
 ++++
 
-Often users will input `date`, `ip`, `version`, `date_period` or `time_duration` as simple strings in their queries for use in predicates, functions, or expressions. {esql} provides <<esql-type-conversion-functions, type conversion functions>> to explicitly convert these strings into the desired data types.
+Often users will input `date`, `date_period`, `time_duration`, `ip` or `version` as simple strings in their queries for use in predicates, functions, or expressions. {esql} provides <<esql-type-conversion-functions, type conversion functions>> to explicitly convert these strings into the desired data types.
 
 Without implicit casting users must explicitly code these `to_X` functions in their queries, when string literals don't match the target data types they are assigned or compared to. Here is an example of using `to_datetime` to explicitly perform a data type conversion.
 
@@ -18,7 +18,10 @@ FROM employees
 | LIMIT 1
 ----
 
-Implicit casting improves usability, by automatically converting string literals to the target data type. This is most useful when the target data type is `date`, `ip`, `version`, `date_period` or `time_duration`. It is natural to specify these as a string in queries.
+[discrete]
+[[esql-implicit-casting-example]]
+==== Implicit casting example
+Implicit casting automatically converts string literals to the target data type. This allows users to specify string values for types like `date`, `date_period`, `time_duration`, `ip` and `version` in their queries.
 
 The first query can be coded without calling the `to_datetime` function, as follows:
 
@@ -31,35 +34,36 @@ FROM employees
 | LIMIT 1
 ----
 
-[float]
-=== Implicit casting support
+[discrete]
+[[esql-implicit-casting-supported-operations]]
+==== Operations that support implicit casting
 
 The following table details which {esql} operations support implicit casting for different data types.
 
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
-||ScalarFunction*|Operator*|<<esql-group-functions, GroupingFunction>>|<<esql-agg-functions, AggregateFunction>>
-|DATE|Y|Y|Y|N
-|IP|Y|Y|Y|N
-|VERSION|Y|Y|Y|N
-|BOOLEAN|Y|Y|Y|N
-|DATE_PERIOD/TIME_DURATION|Y|N|Y|N
+|ScalarFunctions|Operators|<<esql-group-functions, GroupingFunctions>>|<<esql-agg-functions, AggregateFunctions>>
+DATE|Y|Y|Y|N
+DATE_PERIOD/TIME_DURATION|Y|N|Y|N
+IP|Y|Y|Y|N
+VERSION|Y|Y|Y|N
+BOOLEAN|Y|Y|Y|N
 |===
 
-ScalarFunction* includes:
+ScalarFunctions includes:
 
-<<esql-conditional-functions-and-expressions, Conditional Functions and Expressions>>
+* <<esql-conditional-functions-and-expressions, Conditional Functions and Expressions>>
 
-<<esql-date-time-functions, Date and Time Functions>>
+* <<esql-date-time-functions, Date and Time Functions>>
 
-<<esql-ip-functions, IP Functions>>
+* <<esql-ip-functions, IP Functions>>
 
 
-Operator* includes:
+Operators includes:
 
-<<esql-binary-operators, Binary Operators>>
+* <<esql-binary-operators, Binary Operators>>
 
-<<esql-unary-operators, Unary Operator>>
+* <<esql-unary-operators, Unary Operator>>
 
-<<esql-in-operator, IN>>
+* <<esql-in-operator, IN>>