Merge remote-tracking branch 'upstream/main' into bump_compat_api

.buildkite/pipelines/periodic.yml

@@ -584,7 +584,7 @@ steps:
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
       diskSizeGb: 250
-    if: build.branch == "main" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.github/CODEOWNERS

@@ -70,7 +70,3 @@ server/src/main/java/org/elasticsearch/threadpool @elastic/es-core-infra
 # Security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege @elastic/es-security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java @elastic/es-security
-
-# Analytical engine
-x-pack/plugin/esql @elastic/es-analytical-engine
-x-pack/plugin/esql-core @elastic/es-analytical-engine

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip
+distributionSha256Sum=fdfca5dbc2834f0ece5020465737538e5ba679deeff5ab6c09621d67f8bb1a15
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -168,7 +168,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
             vmParameters = [
               '-ea',
               '-Djava.security.manager=allow',
-              '-Djava.locale.providers=SPI,CLDR',
+              '-Djava.locale.providers=CLDR',
               '-Des.nativelibs.path="' + testLibraryPath + '"',
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -93,7 +93,7 @@ public void execute(Task t) {
                     mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile());
 
                     // TODO remove once jvm.options are added to test system properties
-                    test.systemProperty("java.locale.providers", "SPI,CLDR");
+                    test.systemProperty("java.locale.providers", "CLDR");
                 }
             });
             test.getJvmArgumentProviders().add(nonInputProperties);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -59,7 +59,6 @@ public class InternalDistributionModuleCheckTaskProvider {
         "org.elasticsearch.nativeaccess",
         "org.elasticsearch.plugin",
         "org.elasticsearch.plugin.analysis",
-        "org.elasticsearch.pluginclassloader",
         "org.elasticsearch.securesm",
         "org.elasticsearch.server",
         "org.elasticsearch.simdvec",

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java

@@ -22,7 +22,9 @@
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.compile.CompileOptions;
 import org.gradle.api.tasks.compile.JavaCompile;
+import org.gradle.api.tasks.javadoc.Javadoc;
 import org.gradle.api.tasks.testing.Test;
+import org.gradle.external.javadoc.CoreJavadocOptions;
 import org.gradle.jvm.tasks.Jar;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainService;
@@ -73,8 +75,10 @@ public void apply(Project project) {
             List<Integer> mainVersions = findSourceVersions(project);
             List<String> mainSourceSets = new ArrayList<>();
             mainSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);
+            configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.MAIN_SOURCE_SET_NAME), 21);
             List<String> testSourceSets = new ArrayList<>(mainSourceSets);
             testSourceSets.add(SourceSet.TEST_SOURCE_SET_NAME);
+            configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.TEST_SOURCE_SET_NAME), 21);
             for (int javaVersion : mainVersions) {
                 String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
                 SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion);
@@ -124,11 +128,8 @@ private SourceSet addSourceSet(
             compileTask.setSourceCompatibility(Integer.toString(javaVersion));
             CompileOptions compileOptions = compileTask.getOptions();
             compileOptions.getRelease().set(javaVersion);
-            compileOptions.getCompilerArgs().add("--enable-preview");
-            compileOptions.getCompilerArgs().add("-Xlint:-preview");
-
-            compileTask.doLast(t -> { stripPreviewFromFiles(compileTask.getDestinationDirectory().getAsFile().get().toPath()); });
         });
+        configurePreviewFeatures(project, sourceSet, javaVersion);
 
         // Since we configure MRJAR sourcesets to allow preview apis, class signatures for those
         // apis are not known by forbidden apis, so we must ignore all missing classes. We could, in theory,
@@ -142,6 +143,21 @@ private SourceSet addSourceSet(
         return sourceSet;
     }
 
+    private void configurePreviewFeatures(Project project, SourceSet sourceSet, int javaVersion) {
+        project.getTasks().withType(JavaCompile.class).named(sourceSet.getCompileJavaTaskName()).configure(compileTask -> {
+            CompileOptions compileOptions = compileTask.getOptions();
+            compileOptions.getCompilerArgs().add("--enable-preview");
+            compileOptions.getCompilerArgs().add("-Xlint:-preview");
+
+            compileTask.doLast(t -> { stripPreviewFromFiles(compileTask.getDestinationDirectory().getAsFile().get().toPath()); });
+        });
+        project.getTasks().withType(Javadoc.class).named(name -> name.equals(sourceSet.getJavadocTaskName())).configureEach(javadocTask -> {
+            CoreJavadocOptions options = (CoreJavadocOptions) javadocTask.getOptions();
+            options.addBooleanOption("-enable-preview", true);
+            options.addStringOption("-release", String.valueOf(javaVersion));
+        });
+    }
+
     private void configureSourceSetInJar(Project project, SourceSet sourceSet, int javaVersion) {
         var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME);
         jarTask.configure(task -> task.into("META-INF/versions/" + javaVersion, copySpec -> copySpec.from(sourceSet.getOutput())));

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-8.10
+8.10.1

distribution/tools/entitlement-agent/README.md

@@ -0,0 +1,10 @@
+### Entitlement Agent
+
+This is a java agent that instruments sensitive class library methods with calls into the `entitlement-runtime` module to check for permissions granted under the _entitlements_ system.
+
+The entitlements system provides an alternative to the legacy `SecurityManager` system, which is deprecated for removal.
+With this agent, the Elasticsearch server can retain some control over which class library methods can be invoked by which callers.
+
+This module is responsible for inserting the appropriate bytecode to achieve enforcement of the rules governed by the `entitlement-runtime` module.
+
+It is not responsible for permission granting or checking logic. That responsibility lies with `entitlement-runtime`.

distribution/tools/entitlement-agent/build.gradle

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+apply plugin: 'elasticsearch.build'
+
+configurations {
+  entitlementRuntime
+}
+
+dependencies {
+  entitlementRuntime project(":libs:elasticsearch-entitlement-runtime")
+  implementation project(":libs:elasticsearch-entitlement-runtime")
+  testImplementation project(":test:framework")
+}
+
+tasks.named('test').configure {
+  dependsOn('jar')
+  jvmArgs "-javaagent:${ tasks.named('jar').flatMap{ it.archiveFile }.get()}"
+}
+
+tasks.named('jar').configure {
+  manifest {
+    attributes(
+      'Premain-Class': 'org.elasticsearch.entitlement.agent.EntitlementAgent'
+      , 'Can-Retransform-Classes': 'true'
+    )
+  }
+}
+
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}
+

libs/plugin-classloader/src/main/java/module-info.java → distribution/tools/entitlement-agent/src/main/java/module-info.java

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-module org.elasticsearch.pluginclassloader {
-    exports org.elasticsearch.plugins.loader;
+module org.elasticsearch.entitlement.agent {
+    requires java.instrument;
+    requires org.elasticsearch.entitlement.runtime;
 }

libs/plugin-classloader/build.gradle → distribution/tools/entitlement-agent/src/main/java/org/elasticsearch/entitlement/agent/EntitlementAgent.java

@@ -7,12 +7,15 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-// This is only required because :server needs this at runtime.
-// We'll be removing this in 8.0 so for now just publish the JAR to make dependency resolution work.
-apply plugin: 'elasticsearch.publish'
+package org.elasticsearch.entitlement.agent;
 
-tasks.named("test").configure { enabled = false }
+import org.elasticsearch.entitlement.runtime.api.EntitlementChecks;
 
-// test depend on ES core...
-tasks.named('forbiddenApisMain').configure { enabled = false}
-tasks.named("jarHell").configure { enabled = false }
+import java.lang.instrument.Instrumentation;
+
+public class EntitlementAgent {
+
+    public static void premain(String agentArgs, Instrumentation inst) throws Exception {
+        EntitlementChecks.setAgentBooted();
+    }
+}

distribution/tools/entitlement-agent/src/test/java/org/elasticsearch/entitlement/agent/EntitlementAgentTests.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.agent;
+
+import org.elasticsearch.entitlement.runtime.api.EntitlementChecks;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
+
+/**
+ * This is an end-to-end test that runs with the javaagent installed.
+ * It should exhaustively test every instrumented method to make sure it passes with the entitlement
+ * and fails without it.
+ * See {@code build.gradle} for how we set the command line arguments for this test.
+ */
+@WithoutSecurityManager
+public class EntitlementAgentTests extends ESTestCase {
+
+    public void testAgentBooted() {
+        assertTrue(EntitlementChecks.isAgentBooted());
+    }
+
+}

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -11,7 +11,6 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.core.UpdateForV9;
 
 import java.util.List;
 import java.util.Map;
@@ -61,7 +60,7 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dlog4j.shutdownHookEnabled=false",
                 "-Dlog4j2.disable.jmx=true",
                 "-Dlog4j2.formatMsgNoLookups=true",
-                "-Djava.locale.providers=" + getLocaleProviders(),
+                "-Djava.locale.providers=CLDR",
                 maybeEnableNativeAccess(),
                 maybeOverrideDockerCgroup(distroType),
                 maybeSetActiveProcessorCount(nodeSettings),
@@ -73,16 +72,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
         ).filter(e -> e.isEmpty() == false).collect(Collectors.toList());
     }
 
-    @UpdateForV9    // only use CLDR in v9+
-    private static String getLocaleProviders() {
-        /*
-         * Specify SPI to load IsoCalendarDataProvider (see #48209), specifying the first day of week as Monday.
-         * When on pre-23, use COMPAT instead to maintain existing date formats as much as we can.
-         * When on JDK 23+, use the default CLDR locale database, as COMPAT was removed in JDK 23.
-         */
-        return Runtime.version().feature() >= 23 ? "SPI,CLDR" : "SPI,COMPAT";
-    }
-
     /*
      * The virtual file /proc/self/cgroup should list the current cgroup
      * membership. For each hierarchy, you can follow the cgroup path from

docs/changelog/112645.yaml

@@ -0,0 +1,6 @@
+pr: 112645
+summary: Add support for multi-value dimensions
+area: Mapping
+type: enhancement
+issues:
+ - 110387

docs/changelog/112706.yaml

@@ -0,0 +1,5 @@
+pr: 112706
+summary: Configure keeping source in `FieldMapper`
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/112768.yaml

@@ -0,0 +1,5 @@
+pr: 112768
+summary: Deduplicate Kuromoji User Dictionary
+area: Search
+type: enhancement
+issues: []

docs/changelog/112972.yaml

@@ -0,0 +1,6 @@
+pr: 112972
+summary: "ILM: Add `total_shards_per_node` setting to searchable snapshot"
+area: ILM+SLM
+type: enhancement
+issues:
+ - 112261

docs/changelog/113027.yaml

@@ -0,0 +1,6 @@
+pr: 113027
+summary: Retrieve the source for objects and arrays in a separate parsing phase
+area: Mapping
+type: bug
+issues:
+ - 112374

docs/changelog/113102.yaml

@@ -0,0 +1,5 @@
+pr: 113102
+summary: Trigger merges after recovery
+area: Recovery
+type: enhancement
+issues: []

docs/changelog/113103.yaml

@@ -0,0 +1,6 @@
+pr: 113103
+summary: "ESQL: Align year diffing to the rest of the units in DATE_DIFF: chronological"
+area: ES|QL
+type: bug
+issues:
+ - 112482

docs/changelog/113123.yaml

@@ -0,0 +1,6 @@
+pr: 113123
+summary: "ES|QL: Skip CASE function from `InferIsNotNull` rule checks"
+area: ES|QL
+type: bug
+issues:
+ - 112704

docs/changelog/113172.yaml

@@ -0,0 +1,6 @@
+pr: 113172
+summary: "[ESQL] Add finish() elapsed time to aggregation profiling times"
+area: ES|QL
+type: enhancement
+issues:
+ - 112950

docs/changelog/113183.yaml

@@ -0,0 +1,6 @@
+pr: 113183
+summary: "ESQL: TOP support for strings"
+area: ES|QL
+type: feature
+issues:
+ - 109849

docs/changelog/113280.yaml

@@ -0,0 +1,5 @@
+pr: 113280
+summary: Warn for model load failures if they have a status code <500
+area: Machine Learning
+type: bug
+issues: []

docs/internal/DistributedArchitectureGuide.md

@@ -252,7 +252,7 @@ changes. The cloud service will add more resources to the cluster based on Elast
 Elasticsearch by itself cannot automatically scale.
 
 Autoscaling recommendations are tailored for the user [based on user defined policies][], composed of data
-roles (hot, frozen, etc) and [deciders][]. There's a public [webinar on autoscaling][], as well as the
+roles (hot, frozen, etc.) and [deciders][]. There's a public [webinar on autoscaling][], as well as the
 public [Autoscaling APIs] docs.
 
 Autoscaling's current implementation is based primary on storage requirements, as well as memory capacity
@@ -332,7 +332,7 @@ problems in the cluster. It uses [an algorithm defined here][]. Some examples ar
 [an algorithm defined here]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java#L158-L176
 
 The `ProactiveStorageDeciderService` maintains a forecast window that [defaults to 30 minutes][]. It only
-runs on data streams (ILM, rollover, etc), not regular indexes. It looks at past [index changes][] that
+runs on data streams (ILM, rollover, etc.), not regular indexes. It looks at past [index changes][] that
 took place within the forecast window to [predict][] resources that will be needed shortly.
 
 [defaults to 30 minutes]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java#L32

docs/plugins/analysis-icu.asciidoc

@@ -380,7 +380,7 @@ GET /my-index-000001/_search <3>
 
 --------------------------
 
-<1> The `name` field uses the `standard` analyzer, and so support full text queries.
+<1> The `name` field uses the `standard` analyzer, and so supports full text queries.
 <2> The `name.sort` field is an `icu_collation_keyword` field that will preserve the name as
     a single token doc values, and applies the German ``phonebook'' order.
 <3> An example query which searches the `name` field and sorts on the `name.sort` field.
@@ -467,7 +467,7 @@ differences.
 `case_first`::
 
 Possible values: `lower` or `upper`. Useful to control which case is sorted
-first when case is not ignored for strength `tertiary`. The default depends on
+first when the case is not ignored for strength `tertiary`. The default depends on
 the collation.
 
 `numeric`::