Merge branch '8.11' into elser2-811

.buildkite/pipelines/intake.yml

@@ -40,7 +40,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.14", "8.10.3", "8.11.0"]
+            BWC_VERSION: ["7.17.15", "8.10.4", "8.11.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1056,6 +1056,22 @@ steps:
         env:
           BWC_VERSION: 7.17.14
 
+      - label: "{{matrix.image}} / 7.17.15 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.15
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.15
+
       - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
         timeout_in_minutes: 300
@@ -1648,6 +1664,22 @@ steps:
         env:
           BWC_VERSION: 8.10.3
 
+      - label: "{{matrix.image}} / 8.10.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.4
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.4
+
       - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -642,6 +642,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.14
+      - label: 7.17.15 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.15#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.15
       - label: 8.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest
         timeout_in_minutes: 300
@@ -1012,6 +1022,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.10.3
+      - label: 8.10.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.4#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.4
       - label: 8.11.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
         timeout_in_minutes: 300
@@ -1176,6 +1196,7 @@ steps:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
           machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
       - label: third-party / s3
         command: |
           export amazon_s3_bucket=elasticsearch-ci.us-west-2

.ci/bwcVersions

@@ -63,6 +63,7 @@ BWC_VERSION:
   - "7.17.12"
   - "7.17.13"
   - "7.17.14"
+  - "7.17.15"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"
@@ -100,4 +101,5 @@ BWC_VERSION:
   - "8.10.1"
   - "8.10.2"
   - "8.10.3"
+  - "8.10.4"
   - "8.11.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
-  - "7.17.14"
-  - "8.10.3"
+  - "7.17.15"
+  - "8.10.4"
   - "8.11.0"

docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc

@@ -40,8 +40,8 @@ Post an event to an Analytics Collection.
 
 [[post-analytics-collection-event-request-body]]
 ==== {api-request-body-title}
-Full request body parameters can be found in {enterprise-search-ref}/analytics-events-reference.html[Enterprise Analytics Events^].
 
+Full request body parameters can be found in: <<behavioral-analytics-event-reference>>.
 
 [[post-analytics-collection-event-prereqs]]
 ==== {api-prereq-title}

docs/reference/ingest.asciidoc

@@ -448,11 +448,11 @@ configuration. See {fleet-guide}/install-standalone-elastic-agent.html[Install s
 
 [discrete]
 [[pipelines-in-enterprise-search]]
-=== Pipelines in Enterprise Search
+=== Pipelines for search indices
 
-When you create Elasticsearch indices for {enterprise-search-ref}/index.html[Enterprise Search^] use cases, for example, using the {enterprise-search-ref}/crawler.html[web crawler^] or {enterprise-search-ref}/connectors.html[connectors^], these indices are automatically set up with specific ingest pipelines. 
+When you create Elasticsearch indices for search use cases, for example, using the {enterprise-search-ref}/crawler.html[web crawler^] or {enterprise-search-ref}/connectors.html[connectors^], these indices are automatically set up with specific ingest pipelines.
 These processors help optimize your content for search.
-Refer to the {enterprise-search-ref}/ingest-pipelines.html[Enterprise Search documentation^] for more information.
+See <<ingest-pipeline-search>> for more information.
 
 [discrete]
 [[access-source-fields]]

docs/reference/release-notes/8.10.0.asciidoc

@@ -3,6 +3,36 @@
 
 Also see <<breaking-changes-8.10,Breaking changes in 8.10>>.
 
+[[known-issues-8.10.0]]
+[float]
+=== Known issues
+
+// tag::repositorydata-format-change[]
+* Snapshot-based downgrades
++
+The snapshot repository format changed in a manner that prevents earlier
+versions of Elasticsearch from reading the repository contents if it contains
+snapshots from this version and the last cluster to write to this repository
+was in the 8.10 series. This will prevent you from reverting an upgrade to the
+8.10 series by restoring a snapshot taken before the upgrade.
++
+Snapshot repositories written by clusters running versions 8.11.0 and later are
+compatible with all earlier versions. Moreover, clusters running version 8.11.0
+or later will also automatically repair the repository format the first time
+they write to the repository to take or delete a snapshot, making it so that
+all earlier versions can read its contents again.
++
+If you wish to downgrade to a version prior to 8.9.0, take or delete a snapshot
+using a cluster running version 8.11.0 or later to repair the repository format
+first. If you cannot repair the repository in this way, first delete all the
+snapshots in the repository taken with version 8.9.0 or later. To do this will
+require using a cluster running version 8.10.0 or later.
++
+If you wish to downgrade to a version in the 8.9 series, you must take or
+delete a snapshot using a cluster running version 8.11.0 or later to repair the
+repository format first.
+// end::repositorydata-format-change[]
+
 [[breaking-8.10.0]]
 [float]
 === Breaking changes

docs/reference/release-notes/8.10.1.asciidoc

@@ -3,6 +3,12 @@
 
 Also see <<breaking-changes-8.10,Breaking changes in 8.10>>.
 
+[[known-issues-8.10.1]]
+[float]
+=== Known issues
+
+include::8.10.0.asciidoc[tag=repositorydata-format-change]
+
 [[bug-8.10.1]]
 [float]
 === Bug fixes

docs/reference/release-notes/8.10.2.asciidoc

@@ -1,6 +1,10 @@
 [[release-notes-8.10.2]]
 == {es} version 8.10.2
 
-8.10.2 contains no significant changes.
+[[known-issues-8.10.2]]
+[float]
+=== Known issues
+
+include::8.10.0.asciidoc[tag=repositorydata-format-change]
 
 Also see <<breaking-changes-8.10,Breaking changes in 8.10>>.

docs/reference/release-notes/8.10.3.asciidoc

@@ -1,7 +1,11 @@
 [[release-notes-8.10.3]]
 == {es} version 8.10.3
 
-coming[8.10.3]
+[[known-issues-8.10.3]]
+[float]
+=== Known issues
+
+include::8.10.0.asciidoc[tag=repositorydata-format-change]
 
 Also see <<breaking-changes-8.10,Breaking changes in 8.10>>.
 

modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/InstrumentsConcurrencyTests.java

@@ -88,6 +88,7 @@ public ObservableLongCounter buildWithCallback(Consumer<ObservableLongMeasuremen
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100587")
     public void testLockingWhenRegistering() throws Exception {
         Instruments instruments = new Instruments(lockingMeter);
 

rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json

@@ -5,7 +5,7 @@
       "description":"Executes an ESQL request"
     },
     "stability":"experimental",
-    "visibility":"private",
+    "visibility":"public",
     "headers":{
       "accept": [ "application/json"],
       "content_type": ["application/json"]
@@ -32,7 +32,7 @@
       }
     },
     "body":{
-      "description":"Use the `query` element to start a query. Use `time_zone` to specify an execution time zone and 'columnar' to format the answer.",
+      "description":"Use the `query` element to start a query. Use `time_zone` to specify an execution time zone and `columnar` to format the answer.",
       "required":true
     }
   }

server/src/main/java/org/elasticsearch/Version.java

@@ -114,6 +114,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_7_17_12 = new Version(7_17_12_99);
     public static final Version V_7_17_13 = new Version(7_17_13_99);
     public static final Version V_7_17_14 = new Version(7_17_14_99);
+    public static final Version V_7_17_15 = new Version(7_17_15_99);
     public static final Version V_8_0_0 = new Version(8_00_00_99);
     public static final Version V_8_0_1 = new Version(8_00_01_99);
     public static final Version V_8_1_0 = new Version(8_01_00_99);
@@ -151,6 +152,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_10_1 = new Version(8_10_01_99);
     public static final Version V_8_10_2 = new Version(8_10_02_99);
     public static final Version V_8_10_3 = new Version(8_10_03_99);
+    public static final Version V_8_10_4 = new Version(8_10_04_99);
     public static final Version V_8_11_0 = new Version(8_11_00_99);
     public static final Version CURRENT = V_8_11_0;
 

test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java

@@ -1513,7 +1513,7 @@ public MatchingDirectoryReader(DirectoryReader in, Query query) throws IOExcepti
                 @Override
                 public LeafReader wrap(LeafReader leaf) {
                     try {
-                        final IndexSearcher searcher = newSearcher(leaf, false);
+                        final IndexSearcher searcher = newSearcher(leaf, false, true, false);
                         searcher.setQueryCache(null);
                         final Weight weight = searcher.createWeight(query, ScoreMode.COMPLETE_NO_SCORES, 1.0f);
                         final Scorer scorer = weight.scorer(leaf.getContext());