Merge branch 'main' into organize_resources

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.3", "8.17.1", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -287,8 +287,8 @@ steps:
         env:
           BWC_VERSION: 8.15.5
 
-      - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2
+      - label: "{{matrix.image}} / 8.16.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -301,7 +301,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.16.2
+          BWC_VERSION: 8.16.3
 
       - label: "{{matrix.image}} / 8.17.1 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.1

.buildkite/pipelines/periodic-platform-support.yml

@@ -63,6 +63,7 @@ steps:
             image:
               - almalinux-8-aarch64
               - ubuntu-2004-aarch64
+              - ubuntu-2404-aarch64
             GRADLE_TASK:
               - checkPart1
               - checkPart2

.buildkite/pipelines/periodic.yml

@@ -306,8 +306,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.16.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.2#bwcTest
+      - label: 8.16.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -316,7 +316,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.16.2
+          BWC_VERSION: 8.16.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -448,7 +448,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.3", "8.17.1", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -490,7 +490,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.3", "8.17.1", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -15,7 +15,7 @@ BWC_VERSION:
   - "8.13.4"
   - "8.14.3"
   - "8.15.5"
-  - "8.16.2"
+  - "8.16.3"
   - "8.17.1"
   - "8.18.0"
   - "9.0.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "8.16.2"
+  - "8.16.3"
   - "8.17.1"
   - "8.18.0"
   - "9.0.0"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:1b51ff6dba78c98d3e02b0cd64a8ce3238c7a40408d21e3af12a329d44db6f23",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:bfdeddb33330a281950c2a54adef991dbbe6a42832bc505d13b11beaf50ae73f",
         "-wolfi",
         "apk"
     ),

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java

@@ -16,12 +16,14 @@
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.tasks.TaskProvider;
 
 import java.io.File;
 import java.nio.file.Path;
 
 import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView;
+import static org.elasticsearch.gradle.internal.util.DependenciesUtils.thirdPartyDependenciesView;
 import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
 
 public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
@@ -47,7 +49,6 @@ public TaskProvider<? extends Task> createTask(Project project) {
                 project.getDependencies().add(JDK_JAR_HELL_CONFIG_NAME, elasticsearchCoreProject);
             }
         }
-
         TaskProvider<ExportElasticsearchBuildResourcesTask> resourcesTask = project.getTasks()
             .register("thirdPartyAuditResources", ExportElasticsearchBuildResourcesTask.class);
         Path resourcesDir = project.getBuildDir().toPath().resolve("third-party-audit-config");
@@ -59,9 +60,11 @@ public TaskProvider<? extends Task> createTask(Project project) {
         // usually only one task is created. but this construct makes our integTests easier to setup
         project.getTasks().withType(ThirdPartyAuditTask.class).configureEach(t -> {
             Configuration runtimeConfiguration = project.getConfigurations().getByName("runtimeClasspath");
+            FileCollection runtimeThirdParty = thirdPartyDependenciesView(runtimeConfiguration);
             Configuration compileOnly = project.getConfigurations()
                 .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
-            t.setClasspath(runtimeConfiguration.plus(compileOnly));
+            FileCollection compileOnlyThirdParty = thirdPartyDependenciesView(compileOnly);
+            t.getThirdPartyClasspath().from(runtimeThirdParty, compileOnlyThirdParty);
             t.getJarsToScan()
                 .from(
                     createFileCollectionFromNonTransitiveArtifactsView(
@@ -78,7 +81,7 @@ public TaskProvider<? extends Task> createTask(Project project) {
             t.getJavaHome().set(buildParams.flatMap(params -> params.getRuntimeJavaHome()).map(File::getPath));
             t.setSignatureFile(resourcesDir.resolve("forbidden/third-party-audit.txt").toFile());
             t.getJdkJarHellClasspath().from(jdkJarHellConfig);
-            t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnly));
+            t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnlyThirdParty));
         });
         return audit;
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java

@@ -17,7 +17,6 @@
 import org.gradle.api.JavaVersion;
 import org.gradle.api.file.ArchiveOperations;
 import org.gradle.api.file.ConfigurableFileCollection;
-import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.file.ProjectLayout;
@@ -96,8 +95,6 @@ public abstract class ThirdPartyAuditTask extends DefaultTask {
 
     private final ProjectLayout projectLayout;
 
-    private FileCollection classpath;
-
     @Inject
     public ThirdPartyAuditTask(
         ArchiveOperations archiveOperations,
@@ -198,9 +195,7 @@ public Set<String> getMissingClassExcludes() {
     public abstract Property<JavaVersion> getRuntimeJavaVersion();
 
     @Classpath
-    public FileCollection getClasspath() {
-        return classpath;
-    }
+    public abstract ConfigurableFileCollection getThirdPartyClasspath();
 
     @TaskAction
     public void runThirdPartyAudit() throws IOException {
@@ -345,7 +340,7 @@ private String runForbiddenAPIsCli() throws IOException {
             if (javaHome.isPresent()) {
                 spec.setExecutable(javaHome.get() + "/bin/java");
             }
-            spec.classpath(getForbiddenAPIsClasspath(), classpath);
+            spec.classpath(getForbiddenAPIsClasspath(), getThirdPartyClasspath());
             // Enable explicitly for each release as appropriate. Just JDK 20/21/22/23 for now, and just the vector module.
             if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22) || isJavaVersion(VERSION_23)) {
                 spec.jvmArgs("--add-modules", "jdk.incubator.vector");
@@ -383,7 +378,7 @@ private boolean isJavaVersion(JavaVersion version) {
     private Set<String> runJdkJarHellCheck() throws IOException {
         ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
         ExecResult execResult = execOperations.javaexec(spec -> {
-            spec.classpath(getJdkJarHellClasspath(), classpath);
+            spec.classpath(getJdkJarHellClasspath(), getThirdPartyClasspath());
             spec.getMainClass().set(JDK_JAR_HELL_MAIN_CLASS);
             spec.args(getJarExpandDir());
             spec.setIgnoreExitValue(true);
@@ -402,8 +397,4 @@ private Set<String> runJdkJarHellCheck() throws IOException {
         return new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n")));
     }
 
-    public void setClasspath(FileCollection classpath) {
-        this.classpath = classpath;
-    }
-
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java

@@ -9,12 +9,16 @@
 
 package org.elasticsearch.gradle.internal.util;
 
+import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin;
+
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ResolvableDependencies;
 import org.gradle.api.artifacts.component.ComponentIdentifier;
+import org.gradle.api.artifacts.component.ProjectComponentIdentifier;
 import org.gradle.api.artifacts.result.ResolvedComponentResult;
 import org.gradle.api.artifacts.result.ResolvedDependencyResult;
 import org.gradle.api.file.FileCollection;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.specs.AndSpec;
 import org.gradle.api.specs.Spec;
 
@@ -29,7 +33,7 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
     ) {
         ResolvableDependencies incoming = configuration.getIncoming();
         return incoming.artifactView(viewConfiguration -> {
-            Set<ComponentIdentifier> firstLevelDependencyComponents = incoming.getResolutionResult()
+            Provider<Set<ComponentIdentifier>> firstLevelDependencyComponents = incoming.getResolutionResult()
                 .getRootComponent()
                 .map(
                     rootComponent -> rootComponent.getDependencies()
@@ -39,12 +43,36 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
                         .filter(dependency -> dependency.getSelected() instanceof ResolvedComponentResult)
                         .map(dependency -> dependency.getSelected().getId())
                         .collect(Collectors.toSet())
-                )
-                .get();
+                );
             viewConfiguration.componentFilter(
-                new AndSpec<>(identifier -> firstLevelDependencyComponents.contains(identifier), componentFilter)
+                new AndSpec<>(identifier -> firstLevelDependencyComponents.get().contains(identifier), componentFilter)
             );
         }).getFiles();
     }
 
+    /**
+     * This method gives us an artifact view of a configuration that filters out all
+     * project dependencies that are not shadowed jars.
+     * Basically a thirdparty only view of the dependency tree.
+     */
+    public static FileCollection thirdPartyDependenciesView(Configuration configuration) {
+        ResolvableDependencies incoming = configuration.getIncoming();
+        return incoming.artifactView(v -> {
+            // resolve componentIdentifier for all shadowed project dependencies
+            Provider<Set<ComponentIdentifier>> shadowedDependencies = incoming.getResolutionResult()
+                .getRootComponent()
+                .map(
+                    root -> root.getDependencies()
+                        .stream()
+                        .filter(dep -> dep instanceof ResolvedDependencyResult)
+                        .map(dep -> (ResolvedDependencyResult) dep)
+                        .filter(dep -> dep.getResolvedVariant().getDisplayName() == ShadowBasePlugin.COMPONENT_NAME)
+                        .filter(dep -> dep.getSelected() instanceof ResolvedComponentResult)
+                        .map(dep -> dep.getSelected().getId())
+                        .collect(Collectors.toSet())
+                );
+            // filter out project dependencies if they are not a shadowed dependency
+            v.componentFilter(i -> (i instanceof ProjectComponentIdentifier == false || shadowedDependencies.get().contains(i)));
+        }).getFiles();
+    }
 }

build-tools-internal/src/main/resources/changelog-schema.json

@@ -279,6 +279,7 @@
     "compatibilityChangeArea": {
       "type": "string",
       "enum": [
+        "Aggregations",
         "Analysis",
         "Authorization",
         "Cluster and node setting",
@@ -295,6 +296,7 @@
         "Painless",
         "REST API",
         "Rollup",
+        "Search",
         "System requirement",
         "Transform"
       ]

build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt

@@ -155,10 +155,8 @@ org.elasticsearch.cluster.ClusterState#compatibilityVersions()
 
 @defaultMessage ClusterFeatures#nodeFeatures is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
 org.elasticsearch.cluster.ClusterFeatures#nodeFeatures()
-@defaultMessage ClusterFeatures#allNodeFeatures is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
-org.elasticsearch.cluster.ClusterFeatures#allNodeFeatures()
 @defaultMessage ClusterFeatures#clusterHasFeature is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
-org.elasticsearch.cluster.ClusterFeatures#clusterHasFeature(org.elasticsearch.features.NodeFeature)
+org.elasticsearch.cluster.ClusterFeatures#clusterHasFeature(org.elasticsearch.cluster.node.DiscoveryNodes, org.elasticsearch.features.NodeFeature)
 
 @defaultMessage Do not construct this records outside the source files they are declared in
 org.elasticsearch.cluster.SnapshotsInProgress$ShardSnapshotStatus#<init>(java.lang.String, org.elasticsearch.cluster.SnapshotsInProgress$ShardState, org.elasticsearch.repositories.ShardGeneration, java.lang.String, org.elasticsearch.repositories.ShardSnapshotResult)

docs/Versions.asciidoc

@@ -9,6 +9,7 @@ include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
 :docker-repo:       docker.elastic.co/elasticsearch/elasticsearch
 :docker-image:      {docker-repo}:{version}
+:docker-wolfi-image: {docker-repo}-wolfi:{version}
 :kib-docker-repo:   docker.elastic.co/kibana/kibana
 :kib-docker-image:  {kib-docker-repo}:{version}
 :plugin_url:      https://artifacts.elastic.co/downloads/elasticsearch-plugins

docs/changelog/116388.yaml

@@ -0,0 +1,5 @@
+pr: 116388
+summary: Add support for partial shard results
+area: EQL
+type: enhancement
+issues: []

docs/changelog/117989.yaml

@@ -0,0 +1,5 @@
+pr: 117989
+summary: ESQL Add esql hash function
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/118143.yaml

@@ -0,0 +1,5 @@
+pr: 118143
+summary: Infrastructure for assuming cluster features in the next major version
+area: "Infra/Core"
+type: feature
+issues: []

docs/changelog/118484.yaml

@@ -0,0 +1,14 @@
+pr: 118484
+summary: Remove date histogram boolean support
+area: Aggregations
+type: breaking
+issues: []
+breaking:
+  title: Remove date histogram boolean support
+  area: Aggregations
+  details: Elasticsearch no longer allows running Date Histogram aggregations
+           over boolean fields. Instead, use Terms aggregation for boolean
+           fields.
+  impact: We expect the impact to be minimal, as this never produced good
+          results, and has been deprecated for years.
+  notable: false

docs/changelog/118544.yaml

@@ -0,0 +1,5 @@
+pr: 118544
+summary: ESQL - Remove restrictions for disjunctions in full text functions
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/118617.yaml

@@ -0,0 +1,5 @@
+pr: 118617
+summary: Add support for `sparse_vector` queries against `semantic_text` fields
+area: "Search"
+type: enhancement
+issues: []

docs/changelog/118671.yaml

@@ -0,0 +1,11 @@
+pr: 118671
+summary: Adjust `random_score` default field to `_seq_no` field
+area: Search
+type: breaking
+issues: []
+breaking:
+  title: Adjust `random_score` default field to `_seq_no` field
+  area: Search
+  details: When providing a 'seed' parameter to a 'random_score' function in the 'function_score' query but NOT providing a 'field', the default 'field' is switched from '_id' to '_seq_no'.
+  impact: The random scoring and ordering may change when providing a 'seed' and not providing a 'field' to a 'random_score' function. 
+  notable: false

docs/changelog/118674.yaml

@@ -0,0 +1,5 @@
+pr: 118674
+summary: Ignore failures from renormalizing buckets in read-only index
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/118681.yaml

@@ -0,0 +1,6 @@
+pr: 118681
+summary: '`ConnectTransportException` returns retryable BAD_GATEWAY'
+area: Network
+type: enhancement
+issues:
+ - 118320

docs/changelog/118697.yaml

@@ -0,0 +1,6 @@
+pr: 118697
+summary: Esql implicit casting for date nanos
+area: ES|QL
+type: enhancement
+issues:
+ - 118476