Merge branch 'main' into capture_additional_s3_metrics

.buildkite/pipelines/intake.yml

@@ -40,7 +40,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.14", "8.10.2", "8.11.0"]
+            BWC_VERSION: ["7.17.14", "8.10.3", "8.11.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1632,6 +1632,22 @@ steps:
         env:
           BWC_VERSION: 8.10.2
 
+      - label: "{{matrix.image}} / 8.10.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.3
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.3
+
       - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -1002,6 +1002,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.10.2
+      - label: 8.10.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.3#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.3
       - label: 8.11.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
         timeout_in_minutes: 300

.ci/bwcVersions

@@ -99,4 +99,5 @@ BWC_VERSION:
   - "8.10.0"
   - "8.10.1"
   - "8.10.2"
+  - "8.10.3"
   - "8.11.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
   - "7.17.14"
-  - "8.10.2"
+  - "8.10.3"
   - "8.11.0"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -67,7 +67,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters");
         map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
         map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-http");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-disabled");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies");

build-tools-internal/src/main/resources/checkstyle_suppressions.xml

@@ -32,7 +32,7 @@
 
   <!-- Intentionally have multi line string for a bulk request, otherwise this needs to fallback to string concatenation  -->
   <suppress files="modules[/\\]data-streams[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]datastreams[/\\]TsdbDataStreamRestIT.java" checks="LineLength" />
-  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
+  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
 
   <!-- Gradle requires inputs to be seriablizable -->
   <suppress files="build-tools-internal[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]internal[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />

build-tools-internal/version.properties

@@ -2,7 +2,7 @@ elasticsearch     = 8.11.0
 lucene            = 9.7.0
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 20.0.2+9@6e380f22cbe7469fa75fb448bd903d8e
+bundled_jdk = 21+35@fd2272bbf8e04c3dbaee13770090416c
 
 # optional dependencies
 spatial4j         = 0.7

docs/changelog/99193.yaml

@@ -0,0 +1,5 @@
+pr: 99193
+summary: Wait for cluster state in recovery
+area: Recovery
+type: enhancement
+issues: []

docs/changelog/99644.yaml

@@ -0,0 +1,6 @@
+pr: 99644
+summary: Add links to docs from failing bootstrap checks
+area: Infra/Node Lifecycle
+type: enhancement
+issues: [99614]
+

docs/changelog/99695.yaml

@@ -0,0 +1,5 @@
+pr: 99695
+summary: "ESQL: Better management of not stored TEXT fiels with synthetic source"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/99724.yaml

@@ -0,0 +1,5 @@
+pr: 99724
+summary: Upgrade bundled JDK to Java 21
+area: Packaging
+type: upgrade
+issues: []

docs/reference/release-notes.asciidoc

@@ -7,6 +7,7 @@
 This section summarizes the changes in each release.
 
 * <<release-notes-8.11.0>>
+* <<release-notes-8.10.2>>
 * <<release-notes-8.10.1>>
 * <<release-notes-8.10.0>>
 * <<release-notes-8.9.2>>
@@ -51,6 +52,7 @@ This section summarizes the changes in each release.
 --
 
 include::release-notes/8.11.0.asciidoc[]
+include::release-notes/8.10.2.asciidoc[]
 include::release-notes/8.10.1.asciidoc[]
 include::release-notes/8.10.0.asciidoc[]
 include::release-notes/8.9.2.asciidoc[]

docs/reference/release-notes/8.10.2.asciidoc

@@ -0,0 +1,6 @@
+[[release-notes-8.10.2]]
+== {es} version 8.10.2
+
+8.10.2 contains no significant changes.
+
+Also see <<breaking-changes-8.10,Breaking changes in 8.10>>.

docs/reference/setup/bootstrap-checks.asciidoc

@@ -221,14 +221,6 @@ releases are not suitable for production. The early-access check detects these
 early-access snapshots. To pass this check, you must start Elasticsearch on a
 release build of the JVM.
 
-=== G1GC check
-
-Early versions of the HotSpot JVM that shipped with JDK 8 are known to
-have issues that can lead to index corruption when the G1GC collector is
-enabled. The versions impacted are those earlier than the version of
-HotSpot that shipped with JDK 8u40. The G1GC check detects these early
-versions of the HotSpot JVM.
-
 === All permission check
 
 The all permission check ensures that the security policy used during bootstrap

docs/reference/troubleshooting/network-timeouts.asciidoc

@@ -1,11 +1,17 @@
 tag::troubleshooting-network-timeouts-gc-vm[]
 * GC pauses are recorded in the GC logs that {es} emits by default, and also
 usually by the `JvmMonitorService` in the main node logs. Use these logs to
-confirm whether or not GC is resulting in delays.
+confirm whether or not the node is experiencing high heap usage with long GC
+pauses. If so, <<high-jvm-memory-pressure,the troubleshooting guide for high
+heap usage>> has some suggestions for further investigation but typically you
+will need to capture a heap dump during a time of high heap usage to fully
+understand the problem.
 
 * VM pauses also affect other processes on the same host. A VM pause also
 typically causes a discontinuity in the system clock, which {es} will report in
-its logs.
+its logs. If you see evidence of other processes pausing at the same time, or
+unexpected clock discontinuities, investigate the infrastructure on which you
+are running {es}.
 end::troubleshooting-network-timeouts-gc-vm[]
 
 tag::troubleshooting-network-timeouts-packet-capture-elections[]

gradle/verification-metadata.xml

@@ -1600,6 +1600,27 @@
             <sha256 value="7e5870fd2e19b87cbd1981c4ff7203897384c2eb104977f40ce4951b40ab433e" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="openjdk_21" name="linux" version="21">
+         <artifact name="linux-21-aarch64.tar.gz">
+            <sha256 value="e8f4ed1a69815ddf56d7da365116eefc1e5a1159396dffee3dd21616a86d5d28" origin="Generated by Gradle"/>
+         </artifact>
+         <artifact name="linux-21-x64.tar.gz">
+            <sha256 value="a30c454a9bef8f46d5f1bf3122830014a8fbe7ac03b5f8729bc3add4b92a1d0a" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="openjdk_21" name="macos" version="21">
+         <artifact name="macos-21-aarch64.tar.gz">
+            <sha256 value="f12e1e0a2dffc847951598f597c8ee60fb0913932f24b2b09c62cfd2f0f4dfb9" origin="Generated by Gradle"/>
+         </artifact>
+         <artifact name="macos-21-x64.tar.gz">
+            <sha256 value="af32e84c11009f72f783fdcdc9917efc277893988f097e198e2576875d1e88c1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="openjdk_21" name="windows" version="21">
+         <artifact name="windows-21-x64.zip">
+            <sha256 value="5434faaf029e66e7ce6e75770ca384de476750984a7d2881ef7686894c4b4944" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.abego.treelayout" name="org.abego.treelayout.core" version="1.0.3">
          <artifact name="org.abego.treelayout.core-1.0.3.jar">
             <sha256 value="fa5e31395c39c2e7d46aca0f81f72060931607b2fa41bd36038eb2cb6fb93326" origin="Generated by Gradle"/>

modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java

@@ -502,7 +502,6 @@ public void testSpecialValueVariable() throws Exception {
         assertThat(stats.getAvg(), equalTo(3.0));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99156")
     public void testStringSpecialValueVariable() throws Exception {
         // i.e. expression script for term aggregations, which is not allowed
         assertAcked(indicesAdmin().prepareCreate("test").setMapping("text", "type=keyword").get());

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionAggregationScript.java

@@ -83,7 +83,7 @@ public void setNextAggregationValue(Object value) {
                 // _value isn't used in script if specialValue == null
                 if (specialValue != null) {
                     if (value instanceof Number) {
-                        specialValue.setValue(((Number) value).doubleValue());
+                        specialValue.setValue(leaf, ((Number) value).doubleValue());
                     } else {
                         throw new GeneralScriptException("Cannot use expression with text variable using " + exprScript);
                     }

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java

@@ -15,20 +15,21 @@
 import org.apache.lucene.search.IndexSearcher;
 
 import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * A {@link DoubleValuesSource} which has a stub {@link DoubleValues} that holds a dynamically replaceable constant double.
  */
 final class ReplaceableConstDoubleValueSource extends DoubleValuesSource {
-    final ReplaceableConstDoubleValues fv;
 
-    ReplaceableConstDoubleValueSource() {
-        fv = new ReplaceableConstDoubleValues();
-    }
+    private final Map<LeafReaderContext, ReplaceableConstDoubleValues> specialValues = new ConcurrentHashMap<>();
 
     @Override
     public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
-        return fv;
+        ReplaceableConstDoubleValues replaceableConstDoubleValues = new ReplaceableConstDoubleValues();
+        specialValues.put(ctx, replaceableConstDoubleValues);
+        return replaceableConstDoubleValues;
     }
 
     @Override
@@ -38,8 +39,12 @@ public boolean needsScores() {
 
     @Override
     public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
-        if (fv.advanceExact(docId)) return Explanation.match((float) fv.doubleValue(), "ReplaceableConstDoubleValues");
-        else return Explanation.noMatch("ReplaceableConstDoubleValues");
+        // TODO where is this explain called? I bet it's never tested, and probably never called.
+        ReplaceableConstDoubleValues fv = specialValues.get(ctx);
+        if (fv.advanceExact(docId)) {
+            return Explanation.match((float) fv.doubleValue(), "ReplaceableConstDoubleValues");
+        }
+        return Explanation.noMatch("ReplaceableConstDoubleValues");
     }
 
     @Override
@@ -52,7 +57,9 @@ public int hashCode() {
         return System.identityHashCode(this);
     }
 
-    public void setValue(double v) {
+    public void setValue(LeafReaderContext ctx, double v) {
+        ReplaceableConstDoubleValues fv = specialValues.get(ctx);
+        assert fv != null : "getValues must be called before setValue for any given leaf reader context";
         fv.setValue(v);
     }
 

qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilBootstrapChecksTests.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.bootstrap;
 
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.node.NodeValidationException;
 import org.elasticsearch.test.AbstractBootstrapCheckTestCase;
@@ -49,7 +50,17 @@ public void tearDown() throws Exception {
 
     public void testEnforceBootstrapChecks() throws NodeValidationException {
         setEsEnforceBootstrapChecks("true");
-        final List<BootstrapCheck> checks = Collections.singletonList(context -> BootstrapCheck.BootstrapCheckResult.failure("error"));
+        final List<BootstrapCheck> checks = Collections.singletonList(new BootstrapCheck() {
+            @Override
+            public BootstrapCheckResult check(BootstrapContext context) {
+                return BootstrapCheck.BootstrapCheckResult.failure("error");
+            }
+
+            @Override
+            public ReferenceDocs referenceDocs() {
+                return ReferenceDocs.BOOTSTRAP_CHECKS;
+            }
+        });
 
         final Logger logger = mock(Logger.class);