Merge remote-tracking branch 'upstream/main' into dynamic_field_in_te…

…mplate

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v8.14.0$" : "main",
+    "^v8.15.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0"]
+            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/lucene-snapshot/run-tests.yml

@@ -32,6 +32,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: custom-32-98304
       buildDirectory: /dev/shm/bk
+  - label: part4
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
       - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"

.buildkite/pipelines/periodic-packaging.yml

@@ -561,6 +561,22 @@ steps:
         env:
           BWC_VERSION: 8.14.0
 
+      - label: "{{matrix.image}} / 8.15.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.15.0
+
   - group: packaging-tests-windows
     steps:
       - label: "{{matrix.image}} / packaging-tests-windows"

.buildkite/pipelines/periodic-platform-support.yml

@@ -47,6 +47,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp
@@ -70,6 +71,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: aws

.buildkite/pipelines/periodic.template.yml

@@ -49,6 +49,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp
@@ -89,6 +90,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp

.buildkite/pipelines/periodic.yml

@@ -332,6 +332,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.14.0
+      - label: 8.15.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.15.0
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420
@@ -380,6 +390,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp
@@ -396,7 +407,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0"]
+            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -420,6 +431,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp
@@ -438,7 +450,7 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0"]
+            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -32,3 +32,4 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.3"
   - "8.14.0"
+  - "8.15.0"

.ci/snapshotBwcVersions

@@ -2,3 +2,4 @@ BWC_VERSION:
   - "7.17.21"
   - "8.13.3"
   - "8.14.0"
+  - "8.15.0"

branches.json

@@ -4,6 +4,9 @@
     {
       "branch": "main"
     },
+    {
+      "branch": "8.14"
+    },
     {
       "branch": "8.13"
     },

build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy

@@ -454,11 +454,11 @@ Requires the `manage_ml` cluster privilege. This privilege is included in the
 
 `<job_id>`::
 (Required, string)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection]
 
 `<snapshot_id>`::
 (Required, string)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=snapshot-id]
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=snapshot-id]
 
 [[ml-update-snapshot-request-body]]
 == {api-request-body-title}
@@ -470,7 +470,7 @@ The following properties can be updated after the model snapshot is created:
 
 `retain`::
 (Optional, Boolean)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=retain]
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=retain]
 
 
 [[ml-update-snapshot-example]]

build-tools-internal/version.properties

@@ -1,4 +1,4 @@
-elasticsearch     = 8.14.0
+elasticsearch     = 8.15.0
 lucene            = 9.10.0
 
 bundled_jdk_vendor = openjdk

docs/changelog/106133.yaml

@@ -1,5 +1,19 @@
 pr: 106133
-summary: Add an optimised vector distance function for aarch64
+summary: Add a SIMD (Neon) optimised vector distance function for int8
 area: Search
 type: enhancement
 issues: []
+highlight:
+  title: A SIMD (Neon) optimised vector distance function for merging int8 Scalar Quantized vectors has been added
+  body: |-
+    An optimised int8 vector distance implementation for aarch64 has been added.
+    This implementation is currently only used during merging.
+    The vector distance implementation outperforms Lucene's Pamana Vector
+    implementation for binary comparisons by approx 5x (depending on the number
+    of dimensions). It does so by means of SIMD (Neon) intrinsics compiled into a
+    separate native library and link by Panama's FFI. Comparisons are performed on
+    off-heap mmap'ed vector data.
+    Macro benchmarks, SO_Dense_Vector with scalar quantization enabled, shows
+    significant improvements in merge times, approximately 3 times faster. 
+  notable: true
+

docs/changelog/107178.yaml

@@ -0,0 +1,5 @@
+pr: 107178
+summary: "Add support for Azure OpenAI embeddings to inference service"
+area: Machine Learning
+type: feature
+issues: [ ]

docs/changelog/107328.yaml

@@ -0,0 +1,7 @@
+pr: 107328
+summary: "ESQL: Fix missing refs due to pruning renamed grouping columns"
+area: ES|QL
+type: bug
+issues:
+ - 107083
+ - 107166

docs/changelog/107334.yaml

@@ -0,0 +1,5 @@
+pr: 107334
+summary: Adding `cache_stats` to geoip stats API
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/107449.yaml

@@ -0,0 +1,5 @@
+pr: 107449
+summary: Leverage ordinals in enrich lookup
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/107494.yaml

@@ -0,0 +1,6 @@
+pr: 107494
+summary: Handle infinity during synthetic source construction for scaled float field
+area: Mapping
+type: bug
+issues:
+ - 107101

docs/changelog/107517.yaml

@@ -0,0 +1,5 @@
+pr: 107517
+summary: Add GET `_inference` for all inference endpoints
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/107533.yaml

@@ -0,0 +1,5 @@
+pr: 107533
+summary: Add setting for max connections to S3
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/107551.yaml

@@ -0,0 +1,5 @@
+pr: 107551
+summary: Avoid attempting to load the same empty field twice in fetch phase
+area: Search
+type: bug
+issues: []

docs/reference/aggregations/bucket/categorize-text-aggregation.asciidoc

@@ -40,15 +40,15 @@ is an object it has the following properties:
 =====
 `char_filter`::::
 (array of strings or objects)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=char-filter]
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=char-filter]
 
 `tokenizer`::::
 (string or object)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=tokenizer]
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=tokenizer]
 
 `filter`::::
 (array of strings or objects)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=filter]
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=filter]
 =====
 
 `categorization_filters`::

docs/reference/aggregations/pipeline/inference-bucket-aggregation.asciidoc

@@ -65,23 +65,23 @@ aggregations some options can be overridden for each of the two types of model.
 
 `num_top_feature_importance_values`::
 (Optional, integer)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num-top-feature-importance-values]
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num-top-feature-importance-values]
 
 [discrete]
 [[inference-agg-classification-opt]]
 ===== Configuration options for {classification} models
 
 `num_top_classes`::
 (Optional, integer)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes]
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes]
 
 `num_top_feature_importance_values`::
 (Optional, integer)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-feature-importance-values]
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-feature-importance-values]
 
 `prediction_field_type`::
 (Optional, string)
-include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type]
+include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type]
 
 
 [[inference-bucket-agg-example]]

docs/reference/alias.asciidoc

@@ -308,7 +308,7 @@ POST _aliases
 ----
 // TEST[s/^/PUT _data_stream\/logs-nginx.access-prod\nPUT _data_stream\/logs-my_app-default\n/]
 
-include::{es-repo-dir}/indices/aliases.asciidoc[tag=write-index-defaults]
+include::{es-ref-dir}/indices/aliases.asciidoc[tag=write-index-defaults]
 
 TIP: We recommend using data streams to store append-only time series data. If
 you need to update or delete existing time series data, you can perform update or delete operations

docs/reference/analysis/tokenfilters/stemmer-override-tokenfilter.asciidoc

@@ -48,7 +48,7 @@ Where the file looks like:
 
 [source,stemmer_override]
 --------------------------------------------------
-include::{es-test-dir}/cluster/config/analysis/stemmer_override.txt[]
+include::{elasticsearch-root}/docs/src/test/cluster/config/analysis/stemmer_override.txt[]
 --------------------------------------------------
 
 You can also define the overrides rules inline:

docs/reference/api-conventions.asciidoc

@@ -205,18 +205,18 @@ on a remote cluster is currently not supported. For example, this will throw an
 Multi-target APIs that can target indices support the following query
 string parameters:
 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable]
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable]
 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices]
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices]
 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards]
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards]
 
 The defaults settings for the above parameters depend on the API being used.
 
 Some multi-target APIs that can target indices also support the following query
 string parameter:
 
-include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled]
+include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled]
 
 NOTE: APIs with a single target, such as the <<docs-get,get document API>>, do
 not support multi-target syntax.