Merge remote-tracking branch 'origin/main' into carlosdelest/text_exp…

…ansion_supports_sparse_vector

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=5022b0b25fe182b0e50867e77f484501dba44feeea88f5c1f13b6b4660463640
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-all.zip
+distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy

@@ -55,7 +55,8 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest", '--stacktrace').build()
 
         then:
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
+        // we set the task to be skipped if there are no matching tests in the compatibility test sourceSet
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
@@ -164,7 +165,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
         then:
         result.task(':check').outcome == TaskOutcome.UP_TO_DATE
         result.task(':checkRestCompat').outcome == TaskOutcome.UP_TO_DATE
-        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE
+        result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED
         result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java

@@ -43,7 +43,8 @@
 /**
  * The Java Module Compile Path Plugin, i.e. --module-path, ---module-version
  */
-public class ElasticsearchJavaModulePathPlugin implements Plugin<Project> {
+public abstract class ElasticsearchJavaModulePathPlugin implements Plugin<Project> {
+
     @Override
     public void apply(Project project) {
         project.getPluginManager().apply(JavaPlugin.class);
@@ -53,7 +54,7 @@ public void apply(Project project) {
     // List of root tasks, by name, whose compileJava task should not use the module path. These are test related sources.
     static final Set<String> EXCLUDES = Set.of(":test:framework", ":x-pack:plugin:eql:qa:common");
 
-    static void configureCompileModulePath(Project project) {
+    void configureCompileModulePath(Project project) {
         // first disable Gradle's builtin module path inference
         project.getTasks()
             .withType(JavaCompile.class)
@@ -82,12 +83,10 @@ static void configureCompileModulePath(Project project) {
         }).getIncoming().artifactView(it -> {
             it.componentFilter(cf -> {
                 var visited = new HashSet<ComponentIdentifier>();
-                return walkResolvedComponent(
-                    project,
-                    compileClasspath.getIncoming().getResolutionResult().getRoot(),
-                    isModuleProject,
-                    visited
-                ).anyMatch(cf::equals);
+                ResolvedComponentResult root = compileClasspath.getIncoming().getResolutionResult().getRoot();
+                ComponentIdentifier id = root.getId();
+                String currentBuildPath = ((ProjectComponentIdentifier) id).getBuild().getBuildPath();
+                return walkResolvedComponent(currentBuildPath, project, root, isModuleProject, visited).anyMatch(cf::equals);
             });
         }).getFiles();
 
@@ -116,7 +115,8 @@ public void execute(Task t) {
         });
     }
 
-    static Stream<ComponentIdentifier> walkResolvedComponent(
+    Stream<ComponentIdentifier> walkResolvedComponent(
+        String currentBuildPath,
         Project project,
         ResolvedComponentResult result,
         boolean isModuleDependency,
@@ -133,9 +133,10 @@ static Stream<ComponentIdentifier> walkResolvedComponent(
                     return false;
                 }
                 return isModuleDependency
-                    || (it.getId() instanceof ProjectComponentIdentifier projectId && hasModuleInfoDotJava(project, projectId));
+                    || (it.getId() instanceof ProjectComponentIdentifier projectId
+                        && hasModuleInfoDotJava(currentBuildPath, project, projectId));
             })
-            .flatMap(it -> Stream.concat(walkResolvedComponent(project, it, true, visited), Stream.of(it.getId())));
+            .flatMap(it -> Stream.concat(walkResolvedComponent(currentBuildPath, project, it, true, visited), Stream.of(it.getId())));
     }
 
     static class CompileModulePathArgumentProvider implements CommandLineArgumentProvider, Named {
@@ -176,16 +177,16 @@ public String getName() {
         }
     }
 
-    static boolean hasModuleInfoDotJava(Project project) {
+    boolean hasModuleInfoDotJava(Project project) {
         return getJavaMainSourceSet(project).getJava()
             .getSrcDirs()
             .stream()
             .map(dir -> dir.toPath().resolve("module-info.java"))
             .anyMatch(Files::exists);
     }
 
-    static boolean hasModuleInfoDotJava(Project project, ProjectComponentIdentifier id) {
-        return new File(findProjectIdPath(project, id), "src/main/java/module-info.java").exists();
+    boolean hasModuleInfoDotJava(String currentBuildPath, Project project, ProjectComponentIdentifier id) {
+        return new File(findProjectIdPath(currentBuildPath, project, id), "src/main/java/module-info.java").exists();
     }
 
     static SourceSet getJavaMainSourceSet(Project project) {
@@ -209,13 +210,14 @@ static boolean isIdea() {
         return System.getProperty("idea.sync.active", "false").equals("true");
     }
 
-    static File findProjectIdPath(Project project, ProjectComponentIdentifier id) {
-        if (id.getBuild().isCurrentBuild()) {
+    File findProjectIdPath(String currentBuildPath, Project project, ProjectComponentIdentifier id) {
+        if (id.getBuild().getBuildPath().equals(currentBuildPath)) {
             return project.findProject(id.getProjectPath()).getProjectDir();
         } else {
             // For project dependencies sourced from an included build we have to infer the source project path
-            File includedBuildDir = project.getGradle().includedBuild(id.getBuild().getName()).getProjectDir();
-
+            String buildPath = id.getBuild().getBuildPath();
+            String buildName = buildPath.substring(buildPath.lastIndexOf(':') + 1);
+            File includedBuildDir = project.getGradle().includedBuild(buildName).getProjectDir();
             // We have to account for us renaming the :libs projects here
             String[] pathSegments = id.getProjectPath().split(":");
             if (pathSegments[1].equals("libs")) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestIntegTestTask.java

@@ -17,4 +17,4 @@
  * conventional configured tasks of {@link RestIntegTestTask}
  */
 @CacheableTask
-public class RestIntegTestTask extends StandaloneRestIntegTestTask {}
+public abstract class RestIntegTestTask extends StandaloneRestIntegTestTask {}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java

@@ -221,6 +221,7 @@ public void apply(Project project) {
             testTask.setTestClassesDirs(
                 yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs())
             );
+            testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getAllSource().isEmpty() == false);
             testTask.setClasspath(
                 yamlCompatTestSourceSet.getRuntimeClasspath()
                     // remove the "normal" api and tests

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-8.2
+8.3

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java

@@ -36,7 +36,7 @@
  * {@link Nested} inputs.
  */
 @CacheableTask
-public class StandaloneRestIntegTestTask extends Test implements TestClustersAware, FileSystemOperationsAware {
+public abstract class StandaloneRestIntegTestTask extends Test implements TestClustersAware, FileSystemOperationsAware {
 
     private Collection<ElasticsearchCluster> clusters = new HashSet<>();
     private boolean debugServer = false;

distribution/packages/build.gradle

@@ -43,7 +43,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.0.0"
+   id "com.netflix.nebula.ospackage-base" version "11.5.0"
 }
 
 ['deb', 'rpm'].each { type ->

distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceInstallCommand.java

@@ -8,7 +8,7 @@
 
 package org.elasticsearch.windows.service;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.Build;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.ProcessInfo;
 import org.elasticsearch.cli.Terminal;
@@ -47,15 +47,15 @@ protected String getAdditionalArgs(String serviceId, ProcessInfo pinfo) {
         addArg(
             args,
             "--DisplayName",
-            pinfo.envVars().getOrDefault("SERVICE_DISPLAY_NAME", "Elasticsearch %s (%s)".formatted(Version.CURRENT, serviceId))
+            pinfo.envVars().getOrDefault("SERVICE_DISPLAY_NAME", "Elasticsearch %s (%s)".formatted(Build.current().version(), serviceId))
         );
         addArg(
             args,
             "--Description",
             pinfo.envVars()
                 .getOrDefault(
                     "SERVICE_DESCRIPTION",
-                    String.format(java.util.Locale.ROOT, "Elasticsearch %s Windows Service - https://elastic.co", Version.CURRENT)
+                    String.format(java.util.Locale.ROOT, "Elasticsearch %s Windows Service - https://elastic.co", Build.current().version())
                 )
         );
         addQuotedArg(args, "--Jvm", quote(getJvmDll(getJavaHome(pinfo.sysprops())).toString()));

distribution/tools/windows-service-cli/src/test/java/org/elasticsearch/windows/service/WindowsServiceInstallCommandTests.java

@@ -8,7 +8,7 @@
 
 package org.elasticsearch.windows.service;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.Build;
 import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.core.Strings;
@@ -154,13 +154,15 @@ public void testPidFile() throws Exception {
     }
 
     public void testDisplayName() throws Exception {
-        assertServiceArgs(Map.of("DisplayName", Strings.format("\"Elasticsearch %s (elasticsearch-service-x64)\"", Version.CURRENT)));
+        assertServiceArgs(
+            Map.of("DisplayName", Strings.format("\"Elasticsearch %s (elasticsearch-service-x64)\"", Build.current().version()))
+        );
         envVars.put("SERVICE_DISPLAY_NAME", "my service name");
         assertServiceArgs(Map.of("DisplayName", "\"my service name\""));
     }
 
     public void testDescription() throws Exception {
-        String defaultDescription = Strings.format("\"Elasticsearch %s Windows Service - https://elastic.co\"", Version.CURRENT);
+        String defaultDescription = Strings.format("\"Elasticsearch %s Windows Service - https://elastic.co\"", Build.current().version());
         assertServiceArgs(Map.of("Description", defaultDescription));
         envVars.put("SERVICE_DESCRIPTION", "my description");
         assertServiceArgs(Map.of("Description", "\"my description\""));

docs/changelog/99566.yaml

@@ -0,0 +1,6 @@
+pr: 99566
+summary: Add additional counters to `_clusters` response for all Cluster search states
+area: Search
+type: enhancement
+issues:
+ - 98927

docs/changelog/99694.yaml

@@ -0,0 +1,5 @@
+pr: 99694
+summary: Remove shard data files when they fail to write for snapshot
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/99711.yaml

@@ -0,0 +1,5 @@
+pr: 99711
+summary: "ESQL: Date math for negatives"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/99775.yaml

@@ -0,0 +1,6 @@
+pr: 99775
+summary: Adding support for exist queries to `sparse_vector` fields
+area: Search
+type: enhancement
+issues:
+ - 99319

docs/changelog/99796.yaml

@@ -0,0 +1,6 @@
+pr: 99796
+summary: Support runtime fields in synthetic source
+area: Aggregations
+type: bug
+issues:
+ - 98287

docs/changelog/99868.yaml

@@ -0,0 +1,6 @@
+pr: 99868
+summary: Fix fields API for `geo_point` fields inside other arrays
+area: Search
+type: bug
+issues:
+ - 99781

docs/changelog/99874.yaml

@@ -0,0 +1,6 @@
+pr: 99874
+summary: "ESQL: Use exact attributes for data source extraction"
+area: ES|QL
+type: bug
+issues:
+ - 99183

docs/changelog/99909.yaml

@@ -0,0 +1,5 @@
+pr: 99909
+summary: "[Profiling] Allow to customize the ILM policy"
+area: Application
+type: enhancement
+issues: []

docs/changelog/99914.yaml

@@ -0,0 +1,5 @@
+pr: 99914
+summary: Let `_stats` internally timeout if checkpoint information can not be retrieved
+area: Transform
+type: bug
+issues: []

docs/changelog/99917.yaml

@@ -0,0 +1,5 @@
+pr: 99917
+summary: Improve transform log/audit correctness when stopped
+area: Transform
+type: enhancement
+issues: []

docs/changelog/99946.yaml

@@ -0,0 +1,5 @@
+pr: 99946
+summary: Skip settings validation during desired nodes updates
+area: Distributed
+type: bug
+issues: []

docs/reference/indices/resolve.asciidoc

@@ -68,8 +68,8 @@ for the target data stream, index, or index alias.
 +
 --
 (Required, string) Comma-separated name(s) or index pattern(s) of the
-indices, aliases, and data streams to resolve. Resources on
-<<remote-clusters,remote clusters>> can be specified using the
+indices, aliases, and data streams to resolve, using <<api-multi-index>>.
+Resources on <<remote-clusters,remote clusters>> can be specified using the
 `<cluster>:<name>` syntax.
 --
 
@@ -80,6 +80,18 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards]
 +
 Defaults to `open`.
 
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable]
++
+Defaults to `false`.
+
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices]
++
+Defaults to `true`.
+
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled]
++
+Defaults to `false`.
+
 [[resolve-index-api-example]]
 ==== {api-examples-title}
 

docs/reference/mapping/fields/synthetic-source.asciidoc

@@ -28,7 +28,7 @@ PUT idx
 
 While this on the fly reconstruction is *generally* slower than saving the source
 documents verbatim and loading them at query time, it saves a lot of storage
-space. 
+space.
 
 [[synthetic-source-restrictions]]
 ===== Synthetic `_source` restrictions
@@ -37,12 +37,6 @@ There are a couple of restrictions to be aware of:
 
 * When you retrieve synthetic `_source` content it undergoes minor
 <<synthetic-source-modifications,modifications>> compared to the original JSON.
-* The `params._source` is unavailable in scripts. Instead use the
-{painless}/painless-field-context.html[`doc`] API or the <<script-fields-api,`field` API>>.
-* Runtime fields <<runtime-fields-scriptless,without a script>>, and runtime 
-fields that access `_source` are currently not supported for indices that use 
-synthetic `_source`. Use a scripted runtime field that accesses fields <<modules-scripting-doc-vals,using doc values>> or the 
-<<script-fields-api,`field` API>> instead.
 * Synthetic `_source` can be used with indices that contain only these field
 types:
 

docs/reference/release-notes/8.7.0.asciidoc

@@ -16,6 +16,16 @@ include::8.6.0.asciidoc[tag=reconciliation-imbalance-known-issue]
 Ingest Node::
 * Making `JsonProcessor` stricter so that it does not silently drop data {es-pull}93179[#93179] (issue: {es-issue}92898[#92898])
 
+Indices APIs::
+* The <<indices-resolve-index-api>> API implementation was adjusted to use the
+same index resolution mechanism as other similar APIs, adding support for the
+`ignore_unavailable` and `allow_no_indices` flags and the `_all` meta-index. If
+there are no matching indices then earlier versions of this API would return an
+empty result with the `200 OK` HTTP response code, but from 8.7.0 onwards by
+default it returns an `IndexNotFoundException` with the `404 Not Found` HTTP
+response code. To recover the old behaviour, add the query parameter
+`?ignore_unavailable=true` ({es-pull}92820[#92820]).
+
 [[bug-8.7.0]]
 [float]
 === Bug fixes

docs/reference/rest-api/security/get-builtin-privileges.asciidoc

@@ -98,6 +98,7 @@ A successful call returns an object with "cluster" and "index" fields.
     "manage_watcher",
     "monitor",
     "monitor_data_frame_transforms",
+    "monitor_enrich",
     "monitor_ml",
     "monitor_rollup",
     "monitor_snapshot",