Merge branch 'main' into disable_ml_on_macos_x86_64

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.16.1", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -288,8 +288,8 @@ steps:
         env:
           BWC_VERSION: 8.15.4
 
-      - label: "{{matrix.image}} / 8.16.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.1
+      - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -302,7 +302,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.16.1
+          BWC_VERSION: 8.16.2
 
       - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0

.buildkite/pipelines/periodic.yml

@@ -306,8 +306,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.16.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.1#bwcTest
+      - label: 8.16.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -316,7 +316,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.16.1
+          BWC_VERSION: 8.16.2
         retry:
           automatic:
             - exit_status: "-1"
@@ -448,7 +448,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.16.1", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -490,7 +490,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.16.1", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -15,7 +15,7 @@ BWC_VERSION:
   - "8.13.4"
   - "8.14.3"
   - "8.15.4"
-  - "8.16.1"
+  - "8.16.2"
   - "8.17.0"
   - "8.18.0"
   - "9.0.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "8.16.1"
+  - "8.16.2"
   - "8.17.0"
   - "8.18.0"
   - "9.0.0"

build-tools-internal/src/main/resources/checkstyle.xml

@@ -57,7 +57,7 @@
     unfair. -->
   <module name="LineLength">
     <property name="max" value="140" />
-    <property name="ignorePattern" value="^(?:(?:package|import) .*| *\* *https?://[^ ]+)$" />
+    <property name="ignorePattern" value="^(?:(?:package|import) .*| *\*.*https?://.*)$" />
   </module>
 
   <module name="TreeWalker">

docs/changelog/117243.yaml

@@ -0,0 +1,5 @@
+pr: 117243
+summary: Bump major version for feature migration system indices
+area: Infra/Core
+type: upgrade
+issues: []

docs/reference/cluster/update-settings.asciidoc

@@ -59,8 +59,8 @@ An example of a transient update:
 ====
 We no longer recommend using transient cluster settings. Use persistent cluster
 settings instead. If a cluster becomes unstable, transient settings can clear
-unexpectedly, resulting in a potentially undesired cluster configuration. See
-the <<transient-settings-migration-guide>>.
+unexpectedly, resulting in a potentially undesired cluster configuration.
+// See the <<transient-settings-migration-guide>>.
 ====
 // end::transient-settings-warning[]
 

libs/secure-sm/src/main/java/org/elasticsearch/secure_sm/SecureSM.java

@@ -157,7 +157,9 @@ private static void debugThreadGroups(final ThreadGroup caller, final ThreadGrou
     // Returns true if the given thread is an instance of the JDK's InnocuousThread.
     private static boolean isInnocuousThread(Thread t) {
         final Class<?> c = t.getClass();
-        return c.getModule() == Object.class.getModule() && c.getName().equals("jdk.internal.misc.InnocuousThread");
+        return c.getModule() == Object.class.getModule()
+            && (c.getName().equals("jdk.internal.misc.InnocuousThread")
+                || c.getName().equals("java.util.concurrent.ForkJoinWorkerThread$InnocuousForkJoinWorkerThread"));
     }
 
     protected void checkThreadAccess(Thread t) {
@@ -184,19 +186,29 @@ protected void checkThreadAccess(Thread t) {
     private static final Permission MODIFY_THREADGROUP_PERMISSION = new RuntimePermission("modifyThreadGroup");
     private static final Permission MODIFY_ARBITRARY_THREADGROUP_PERMISSION = new ThreadPermission("modifyArbitraryThreadGroup");
 
+    // Returns true if the given thread is an instance of the JDK's InnocuousThread.
+    private static boolean isInnocuousThreadGroup(ThreadGroup t) {
+        final Class<?> c = t.getClass();
+        return c.getModule() == Object.class.getModule() && t.getName().equals("InnocuousForkJoinWorkerThreadGroup");
+    }
+
     protected void checkThreadGroupAccess(ThreadGroup g) {
         Objects.requireNonNull(g);
 
+        boolean targetThreadGroupIsInnocuous = isInnocuousThreadGroup(g);
+
         // first, check if we can modify thread groups at all.
-        checkPermission(MODIFY_THREADGROUP_PERMISSION);
+        if (targetThreadGroupIsInnocuous == false) {
+            checkPermission(MODIFY_THREADGROUP_PERMISSION);
+        }
 
         // check the threadgroup, if its our thread group or an ancestor, its fine.
         final ThreadGroup source = Thread.currentThread().getThreadGroup();
         final ThreadGroup target = g;
 
         if (source == null) {
             return; // we are a dead thread, do nothing
-        } else if (source.parentOf(target) == false) {
+        } else if (source.parentOf(target) == false && targetThreadGroupIsInnocuous == false) {
             checkPermission(MODIFY_ARBITRARY_THREADGROUP_PERMISSION);
         }
     }

libs/secure-sm/src/test/java/org/elasticsearch/secure_sm/SecureSMTests.java

@@ -14,7 +14,10 @@
 import java.security.Permission;
 import java.security.Policy;
 import java.security.ProtectionDomain;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
 
 /** Simple tests for SecureSM */
 public class SecureSMTests extends TestCase {
@@ -128,4 +131,12 @@ public void run() {
         t1.join();
         assertTrue(interrupted1.get());
     }
+
+    public void testParallelStreamThreadGroup() throws Exception {
+        List<Integer> list = new ArrayList<>();
+        for (int i = 0; i < 100; ++i) {
+            list.add(i);
+        }
+        list.parallelStream().collect(Collectors.toSet());
+    }
 }

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/adjacency/InternalAdjacencyMatrix.java

@@ -81,14 +81,12 @@ public InternalAggregations getAggregations() {
             return aggregations;
         }
 
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        private void bucketToXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             builder.field(CommonFields.KEY.getPreferredName(), key);
             builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
             aggregations.toXContentInternal(builder, params);
             builder.endObject();
-            return builder;
         }
 
         @Override
@@ -237,7 +235,7 @@ public InternalAggregation finalizeSampling(SamplingContext samplingContext) {
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         builder.startArray(CommonFields.BUCKETS.getPreferredName());
         for (InternalBucket bucket : buckets) {
-            bucket.toXContent(builder, params);
+            bucket.bucketToXContent(builder, params);
         }
         builder.endArray();
         return builder;

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogram.java

@@ -99,8 +99,7 @@ public Object getKey() {
             return Instant.ofEpochMilli(key).atZone(ZoneOffset.UTC);
         }
 
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        private void bucketToXContent(XContentBuilder builder, Params params, DocValueFormat format) throws IOException {
             String keyAsString = format.format(key).toString();
             builder.startObject();
             if (format != DocValueFormat.RAW) {
@@ -110,7 +109,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
             aggregations.toXContentInternal(builder, params);
             builder.endObject();
-            return builder;
         }
 
         @Override
@@ -597,7 +595,7 @@ private BucketReduceResult mergeConsecutiveBuckets(
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         builder.startArray(CommonFields.BUCKETS.getPreferredName());
         for (Bucket bucket : buckets) {
-            bucket.toXContent(builder, params);
+            bucket.bucketToXContent(builder, params, format);
         }
         builder.endArray();
         builder.field("interval", getInterval().toString());

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java

@@ -36,24 +36,21 @@ public class InternalTimeSeries extends InternalMultiBucketAggregation<InternalT
      */
     public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket {
         protected long bucketOrd;
-        protected final boolean keyed;
         protected final BytesRef key;
         // TODO: make computing docCount optional
         protected long docCount;
         protected InternalAggregations aggregations;
 
-        public InternalBucket(BytesRef key, long docCount, InternalAggregations aggregations, boolean keyed) {
+        public InternalBucket(BytesRef key, long docCount, InternalAggregations aggregations) {
             this.key = key;
             this.docCount = docCount;
             this.aggregations = aggregations;
-            this.keyed = keyed;
         }
 
         /**
          * Read from a stream.
          */
-        public InternalBucket(StreamInput in, boolean keyed) throws IOException {
-            this.keyed = keyed;
+        public InternalBucket(StreamInput in) throws IOException {
             key = in.readBytesRef();
             docCount = in.readVLong();
             aggregations = InternalAggregations.readFrom(in);
@@ -86,8 +83,7 @@ public InternalAggregations getAggregations() {
             return aggregations;
         }
 
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        private void bucketToXContent(XContentBuilder builder, Params params, boolean keyed) throws IOException {
             // Use map key in the xcontent response:
             var key = getKey();
             if (keyed) {
@@ -99,7 +95,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount);
             aggregations.toXContentInternal(builder, params);
             builder.endObject();
-            return builder;
         }
 
         @Override
@@ -112,14 +107,13 @@ public boolean equals(Object other) {
             }
             InternalTimeSeries.InternalBucket that = (InternalTimeSeries.InternalBucket) other;
             return Objects.equals(key, that.key)
-                && Objects.equals(keyed, that.keyed)
                 && Objects.equals(docCount, that.docCount)
                 && Objects.equals(aggregations, that.aggregations);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(getClass(), key, keyed, docCount, aggregations);
+            return Objects.hash(getClass(), key, docCount, aggregations);
         }
     }
 
@@ -143,7 +137,7 @@ public InternalTimeSeries(StreamInput in) throws IOException {
         int size = in.readVInt();
         List<InternalTimeSeries.InternalBucket> buckets = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
-            buckets.add(new InternalTimeSeries.InternalBucket(in, keyed));
+            buckets.add(new InternalTimeSeries.InternalBucket(in));
         }
         this.buckets = buckets;
         this.bucketMap = null;
@@ -162,7 +156,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th
             builder.startArray(CommonFields.BUCKETS.getPreferredName());
         }
         for (InternalBucket bucket : buckets) {
-            bucket.toXContent(builder, params);
+            bucket.bucketToXContent(builder, params, keyed);
         }
         if (keyed) {
             builder.endObject();
@@ -252,14 +246,14 @@ public InternalTimeSeries create(List<InternalBucket> buckets) {
 
     @Override
     public InternalBucket createBucket(InternalAggregations aggregations, InternalBucket prototype) {
-        return new InternalBucket(prototype.key, prototype.docCount, aggregations, prototype.keyed);
+        return new InternalBucket(prototype.key, prototype.docCount, aggregations);
     }
 
     private InternalBucket reduceBucket(List<InternalBucket> buckets, AggregationReduceContext context) {
         InternalTimeSeries.InternalBucket reduced = null;
         for (InternalTimeSeries.InternalBucket bucket : buckets) {
             if (reduced == null) {
-                reduced = new InternalTimeSeries.InternalBucket(bucket.key, bucket.docCount, bucket.aggregations, bucket.keyed);
+                reduced = new InternalTimeSeries.InternalBucket(bucket.key, bucket.docCount, bucket.aggregations);
             } else {
                 reduced.docCount += bucket.docCount;
             }

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java

@@ -83,8 +83,7 @@ public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throw
                     InternalTimeSeries.InternalBucket bucket = new InternalTimeSeries.InternalBucket(
                         BytesRef.deepCopyOf(spare), // Closing bucketOrds will corrupt the bytes ref, so need to make a deep copy here.
                         docCount,
-                        null,
-                        keyed
+                        null
                     );
                     bucket.bucketOrd = ordsEnum.ord();
                     buckets.add(bucket);

modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java

@@ -49,7 +49,7 @@ private List<InternalBucket> randomBuckets(boolean keyed, InternalAggregations a
             }
             try {
                 var key = TimeSeriesIdFieldMapper.buildLegacyTsid(routingPathFields).toBytesRef();
-                bucketList.add(new InternalBucket(key, docCount, aggregations, keyed));
+                bucketList.add(new InternalBucket(key, docCount, aggregations));
             } catch (IOException e) {
                 throw new UncheckedIOException(e);
             }
@@ -108,29 +108,29 @@ public void testReduceSimple() {
         InternalTimeSeries first = new InternalTimeSeries(
             "ts",
             List.of(
-                new InternalBucket(new BytesRef("1"), 3, InternalAggregations.EMPTY, false),
-                new InternalBucket(new BytesRef("10"), 6, InternalAggregations.EMPTY, false),
-                new InternalBucket(new BytesRef("2"), 2, InternalAggregations.EMPTY, false),
-                new InternalBucket(new BytesRef("9"), 5, InternalAggregations.EMPTY, false)
+                new InternalBucket(new BytesRef("1"), 3, InternalAggregations.EMPTY),
+                new InternalBucket(new BytesRef("10"), 6, InternalAggregations.EMPTY),
+                new InternalBucket(new BytesRef("2"), 2, InternalAggregations.EMPTY),
+                new InternalBucket(new BytesRef("9"), 5, InternalAggregations.EMPTY)
             ),
             false,
             Map.of()
         );
         InternalTimeSeries second = new InternalTimeSeries(
             "ts",
             List.of(
-                new InternalBucket(new BytesRef("2"), 1, InternalAggregations.EMPTY, false),
-                new InternalBucket(new BytesRef("3"), 3, InternalAggregations.EMPTY, false)
+                new InternalBucket(new BytesRef("2"), 1, InternalAggregations.EMPTY),
+                new InternalBucket(new BytesRef("3"), 3, InternalAggregations.EMPTY)
             ),
             false,
             Map.of()
         );
         InternalTimeSeries third = new InternalTimeSeries(
             "ts",
             List.of(
-                new InternalBucket(new BytesRef("1"), 2, InternalAggregations.EMPTY, false),
-                new InternalBucket(new BytesRef("3"), 4, InternalAggregations.EMPTY, false),
-                new InternalBucket(new BytesRef("9"), 4, InternalAggregations.EMPTY, false)
+                new InternalBucket(new BytesRef("1"), 2, InternalAggregations.EMPTY),
+                new InternalBucket(new BytesRef("3"), 4, InternalAggregations.EMPTY),
+                new InternalBucket(new BytesRef("9"), 4, InternalAggregations.EMPTY)
             ),
             false,
             Map.of()