Merge remote-tracking branch 'upstream/main' into search_pit_resolved…

…_indices
jimczi · Dec 8, 2023 · a977a1b · a977a1b
2 parents 89853df + 39d9ce8
commit a977a1b
Show file tree

Hide file tree

Showing 246 changed files with 1,992 additions and 1,470 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.16", "8.11.2", "8.12.0", "8.13.0"]
+            BWC_VERSION: ["7.17.16", "8.11.3", "8.12.0", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -1745,6 +1745,22 @@ steps:
         env:
           BWC_VERSION: 8.11.2
 
+      - label: "{{matrix.image}} / 8.11.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.3
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.3
+
       - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0
         timeout_in_minutes: 300

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -1072,6 +1072,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.11.2
+      - label: 8.11.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.3#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.3
       - label: 8.12.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
         timeout_in_minutes: 300

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -106,5 +106,6 @@ BWC_VERSION:
   - "8.11.0"
   - "8.11.1"
   - "8.11.2"
+  - "8.11.3"
   - "8.12.0"
   - "8.13.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.16"
-  - "8.11.2"
+  - "8.11.3"
   - "8.12.0"
   - "8.13.0"
diff --git a/...internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/...internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java
@@ -108,7 +108,10 @@ public void execute(Task t) {
                 "--add-opens=java.base/java.nio.file=ALL-UNNAMED",
                 "--add-opens=java.base/java.time=ALL-UNNAMED",
                 "--add-opens=java.management/java.lang.management=ALL-UNNAMED",
-                "-XX:+HeapDumpOnOutOfMemoryError"
+                "-XX:+HeapDumpOnOutOfMemoryError",
+                // REMOVE once bumped to a JDK greater than 21.0.1, https://github.com/elastic/elasticsearch/issues/103004
+                "-XX:CompileCommand=exclude,org.apache.lucene.util.MSBRadixSorter::computeCommonPrefixLengthAndBuildHistogram",
+                "-XX:CompileCommand=exclude,org.apache.lucene.util.RadixSelector::computeCommonPrefixLengthAndBuildHistogram"
             );
 
             test.getJvmArgumentProviders().add(new SimpleCommandLineArgumentProvider("-XX:HeapDumpPath=" + heapdumpDir));

diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options
@@ -58,6 +58,10 @@
 # result in less optimal vector performance
 20-:--add-modules=jdk.incubator.vector
 
+# REMOVE once bumped to a JDK greater than 21.0.1, https://github.com/elastic/elasticsearch/issues/103004
+19-21:-XX:CompileCommand=exclude,org.apache.lucene.util.MSBRadixSorter::computeCommonPrefixLengthAndBuildHistogram
+19-21:-XX:CompileCommand=exclude,org.apache.lucene.util.RadixSelector::computeCommonPrefixLengthAndBuildHistogram
+
 ## heap dumps
 
 # generate a heap dump when an allocation from the Java heap fails; heap dumps

diff --git a/docs/changelog/100986.yaml b/docs/changelog/100986.yaml
diff --git a/docs/changelog/101915.yaml b/docs/changelog/101915.yaml
diff --git a/docs/changelog/102057.yaml b/docs/changelog/102057.yaml
diff --git a/docs/changelog/102114.yaml b/docs/changelog/102114.yaml
diff --git a/docs/changelog/102151.yaml b/docs/changelog/102151.yaml
diff --git a/docs/changelog/102220.yaml b/docs/changelog/102220.yaml
diff --git a/docs/changelog/102230.yaml b/docs/changelog/102230.yaml
diff --git a/docs/changelog/102240.yaml b/docs/changelog/102240.yaml
diff --git a/docs/changelog/102250.yaml b/docs/changelog/102250.yaml
diff --git a/docs/changelog/102259.yaml b/docs/changelog/102259.yaml
diff --git a/docs/changelog/102281.yaml b/docs/changelog/102281.yaml
diff --git a/docs/changelog/102282.yaml b/docs/changelog/102282.yaml
diff --git a/docs/changelog/102311.yaml b/docs/changelog/102311.yaml
diff --git a/docs/changelog/102396.yaml b/docs/changelog/102396.yaml
diff --git a/docs/changelog/102399.yaml b/docs/changelog/102399.yaml
diff --git a/docs/changelog/102435.yaml b/docs/changelog/102435.yaml
@@ -0,0 +1,5 @@
+pr: 102435
+summary: S3 first byte latency metric
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/102467.yaml b/docs/changelog/102467.yaml
diff --git a/docs/changelog/102492.yaml b/docs/changelog/102492.yaml
diff --git a/docs/changelog/102580.yaml b/docs/changelog/102580.yaml
diff --git a/docs/changelog/102599.yaml b/docs/changelog/102599.yaml
diff --git a/docs/changelog/102715.yaml b/docs/changelog/102715.yaml
diff --git a/docs/changelog/102716.yaml b/docs/changelog/102716.yaml
diff --git a/docs/changelog/102779.yaml b/docs/changelog/102779.yaml
diff --git a/docs/changelog/102821.yaml b/docs/changelog/102821.yaml
diff --git a/docs/changelog/102831.yaml b/docs/changelog/102831.yaml
diff --git a/docs/changelog/102891.yaml b/docs/changelog/102891.yaml
diff --git a/docs/changelog/102934.yaml b/docs/changelog/102934.yaml
diff --git a/docs/changelog/103024.yaml b/docs/changelog/103024.yaml
@@ -0,0 +1,6 @@
+pr: 103024
+summary: Fix template simulate setting application ordering
+area: Indices APIs
+type: bug
+issues:
+ - 103008
diff --git a/docs/changelog/103061.yaml b/docs/changelog/103061.yaml
@@ -0,0 +1,5 @@
+pr: 103061
+summary: "[Profiling] Query in parallel only if beneficial"
+area: Application
+type: bug
+issues: []
diff --git a/docs/changelog/103087.yaml b/docs/changelog/103087.yaml
@@ -0,0 +1,5 @@
+pr: 103087
+summary: Use latest version of entsearch ingestion pipeline
+area: Application
+type: bug
+issues: []
diff --git a/docs/changelog/103112.yaml b/docs/changelog/103112.yaml
@@ -0,0 +1,5 @@
+pr: 103112
+summary: Add JIT compiler excludes for `computeCommonPrefixLengthAndBuildHistogram`
+area: Search
+type: bug
+issues: []
diff --git a/docs/changelog/103124.yaml b/docs/changelog/103124.yaml
@@ -0,0 +1,5 @@
+pr: 103124
+summary: Start a new trace context before loading a trained model
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
@@ -8,6 +8,7 @@ This section summarizes the changes in each release.
 
 * <<release-notes-8.13.0>>
 * <<release-notes-8.12.0>>
+* <<release-notes-8.11.2>>
 * <<release-notes-8.11.1>>
 * <<release-notes-8.11.0>>
 * <<release-notes-8.10.4>>
@@ -58,6 +59,7 @@ This section summarizes the changes in each release.
 
 include::release-notes/8.13.0.asciidoc[]
 include::release-notes/8.12.0.asciidoc[]
+include::release-notes/8.11.2.asciidoc[]
 include::release-notes/8.11.1.asciidoc[]
 include::release-notes/8.11.0.asciidoc[]
 include::release-notes/8.10.4.asciidoc[]

diff --git a/docs/reference/release-notes/8.11.2.asciidoc b/docs/reference/release-notes/8.11.2.asciidoc
@@ -0,0 +1,83 @@
+[[release-notes-8.11.2]]
+== {es} version 8.11.2
+
+Also see <<breaking-changes-8.11,Breaking changes in 8.11>>.
+
+[[known-issues-8.11.2]]
+[float]
+=== Known issues
+include::8.10.3.asciidoc[tag=no-preventive-gc-issue]
+
+[[bug-8.11.2]]
+[float]
+=== Bug fixes
+
+Allocation::
+* Improve failure handling in `ContinuousComputation` {es-pull}102281[#102281]
+
+Application::
+* Default `run_ml_inference` should be true {es-pull}102151[#102151]
+* [Query Rules] Fix bug where combining the same metadata with text/numeric values leads to error {es-pull}102891[#102891] (issue: {es-issue}102827[#102827])
+
+Cluster Coordination::
+* Synchronize Coordinator#onClusterStateApplied {es-pull}100986[#100986] (issue: {es-issue}99023[#99023])
+
+Data streams::
+* [Usage API] Count all the data streams that have lifecycle {es-pull}102259[#102259]
+
+ES|QL::
+* ES|QL: Fix drop of renamed grouping {es-pull}102282[#102282] (issue: {es-issue}102121[#102121])
+* ES|QL: Fix layout management for Project {es-pull}102399[#102399] (issue: {es-issue}102120[#102120])
+* Fix DISSECT with empty patterns {es-pull}102580[#102580] (issue: {es-issue}102577[#102577])
+* Fix leaking blocks in TopN {es-pull}102715[#102715] (issue: {es-issue}102646[#102646])
+* Fix leaking blocks in `BlockUtils` {es-pull}102716[#102716]
+* Fix memory tracking in TopN.Row {es-pull}102831[#102831] (issues: {es-issue}100640[#100640], {es-issue}102784[#102784], {es-issue}102790[#102790], {es-issue}102683[#102683])
+
+ILM+SLM::
+* [ILM] Fix downsample to skip already downsampled indices {es-pull}102250[#102250] (issue: {es-issue}102249[#102249])
+
+Infra/Circuit Breakers::
+* Add more logging to the real memory circuit breaker and lower minimum interval {es-pull}102396[#102396]
+
+Ingest Node::
+* Better processor stat merge {es-pull}102821[#102821]
+
+Machine Learning::
+* Ensure datafeed previews with no start or end time don't search the cold or frozen tiers {es-pull}102492[#102492]
+* Recreate the Elasticsearch private temporary directory if it doesn't exist when an ML job is opened {es-pull}102599[#102599]
+
+Mapping::
+* Fix dense_vector cluster stats indexed_vector_dim_min/max values {es-pull}102467[#102467] (issue: {es-issue}102416[#102416])
+
+Search::
+* Allow mismatched sort-by field types if there are no docs to sort {es-pull}102779[#102779]
+
+Security::
+* Fix double-completion in `SecurityUsageTransportAction` {es-pull}102114[#102114] (issue: {es-issue}102111[#102111])
+
+Snapshot/Restore::
+* Set region for the STS client via privileged calls in AWS SDK {es-pull}102230[#102230] (issue: {es-issue}102173[#102173])
+* Simplify `BlobStoreRepository` idle check {es-pull}102057[#102057] (issue: {es-issue}101948[#101948])
+
+Transform::
+* Ensure transform updates only modify the expected transform task {es-pull}102934[#102934] (issue: {es-issue}102933[#102933])
+* Exclude stack traces from transform audit messages and health {es-pull}102240[#102240]
+
+[[enhancement-8.11.2]]
+[float]
+=== Enhancements
+
+Machine Learning::
+* Add inference counts by model to the machine learning usage stats {es-pull}101915[#101915]
+
+Security::
+* Upgrade xmlsec to 2.3.4 {es-pull}102220[#102220]
+
+[[upgrade-8.11.2]]
+[float]
+=== Upgrades
+
+Snapshot/Restore::
+* Upgrade reactor netty http version {es-pull}102311[#102311]
+
+