Merge branch 'main' into retrievers-rerank

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v8.15.0$" : "main",
+    "^v8.16.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/dra-workflow.yml

@@ -7,7 +7,7 @@ steps:
       image: family/elasticsearch-ubuntu-2204
       machineType: custom-32-98304
       buildDirectory: /dev/shm/bk
-      diskSizeGb: 250
+      diskSizeGb: 350
   - wait
   # The hadoop build depends on the ES artifact
   # So let's trigger the hadoop build any time we build a new staging artifact

.buildkite/pipelines/intake.yml

@@ -62,7 +62,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.23", "8.14.2", "8.15.0"]
+            BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -577,8 +577,8 @@ steps:
         env:
           BWC_VERSION: 8.13.4
 
-      - label: "{{matrix.image}} / 8.14.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.2
+      - label: "{{matrix.image}} / 8.14.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -592,7 +592,7 @@ steps:
           buildDirectory: /dev/shm/bk
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.14.2
+          BWC_VERSION: 8.14.4
 
       - label: "{{matrix.image}} / 8.15.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.0
@@ -611,6 +611,23 @@ steps:
         env:
           BWC_VERSION: 8.15.0
 
+      - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+          diskSizeGb: 250
+        env:
+          BWC_VERSION: 8.16.0
+
   - group: packaging-tests-windows
     steps:
       - label: "{{matrix.image}} / packaging-tests-windows"

.buildkite/pipelines/periodic.yml

@@ -642,8 +642,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.14.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.2#bwcTest
+      - label: 8.14.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -653,7 +653,7 @@ steps:
           preemptible: true
           diskSizeGb: 250
         env:
-          BWC_VERSION: 8.14.2
+          BWC_VERSION: 8.14.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -682,6 +682,26 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 8.16.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+          diskSizeGb: 250
+        env:
+          BWC_VERSION: 8.16.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420
@@ -751,7 +771,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.23", "8.14.2", "8.15.0"]
+            BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -801,7 +821,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.23", "8.14.2", "8.15.0"]
+            BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/scripts/gradle-cache-validation.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+set -euo pipefail
+
+VALIDATION_SCRIPTS_VERSION=2.5.1
+GRADLE_ENTERPRISE_ACCESS_KEY=$(vault kv get -field=value secret/ci/elastic-elasticsearch/gradle-enterprise-api-key)
+export GRADLE_ENTERPRISE_ACCESS_KEY
+
+curl -s -L -O https://github.com/gradle/gradle-enterprise-build-validation-scripts/releases/download/v$VALIDATION_SCRIPTS_VERSION/gradle-enterprise-gradle-build-validation-$VALIDATION_SCRIPTS_VERSION.zip && unzip -q -o gradle-enterprise-gradle-build-validation-$VALIDATION_SCRIPTS_VERSION.zip
+
+# Create a temporary file
+tmpOutputFile=$(mktemp)
+trap "rm $tmpOutputFile" EXIT
+
+gradle-enterprise-gradle-build-validation/03-validate-local-build-caching-different-locations.sh -r https://github.com/elastic/elasticsearch.git -b $BUILDKITE_BRANCH --gradle-enterprise-server https://gradle-enterprise.elastic.co -t precommit --fail-if-not-fully-cacheable | tee $tmpOutputFile
+
+# Capture the return value
+retval=$?
+
+# Now read the content from the temporary file into a variable
+perfOutput=$(cat $tmpOutputFile | sed -n '/Performance Characteristics/,/See https:\/\/gradle.com\/bvs\/main\/Gradle.md#performance-characteristics for details./p' | sed '$d' | sed 's/\x1b\[[0-9;]*m//g')
+investigationOutput=$(cat $tmpOutputFile | sed -n '/Investigation Quick Links/,$p' | sed 's/\x1b\[[0-9;]*m//g')
+
+# Initialize HTML output variable
+summaryHtml="<h4>Performance Characteristics</h4>"
+summaryHtml+="<ul>"
+
+# Process each line of the string
+while IFS=: read -r label value; do
+    if [[ -n "$label" && -n "$value" ]]; then
+        # Trim whitespace from label and value
+        trimmed_label=$(echo "$label" | xargs)
+        trimmed_value=$(echo "$value" | xargs)
+
+        # Append to HTML output variable
+        summaryHtml+="<li><strong>$trimmed_label:</strong> $trimmed_value</li>"
+    fi
+done <<< "$perfOutput"
+
+summaryHtml+="</ul>"
+
+# generate html for links
+summaryHtml+="<h4>Investigation Links</h4>"
+summaryHtml+="<ul>"
+
+# Process each line of the string
+while IFS= read -r line; do
+    if [[ "$line" =~ http.* ]]; then
+        # Extract URL and description using awk
+        url=$(echo "$line" | awk '{print $NF}')
+        description=$(echo "$line" | sed -e "s/:.*//")
+
+        # Append to HTML output variable
+        summaryHtml+="    <li><a href=\"$url\">$description</a></li>"
+    fi
+done <<< "$investigationOutput"
+
+# End of the HTML content
+summaryHtml+="</ul>"
+
+cat << EOF | buildkite-agent annotate --context "ctx-validation-summary" --style "info"
+$summaryHtml
+EOF
+
+# Check if the command was successful
+if [ $retval -eq 0 ]; then
+    echo "Experiment completed successfully"
+elif [ $retval -eq 1 ]; then
+    echo "An invalid input was provided while attempting to run the experiment"
+elif [ $retval -eq 2 ]; then
+    echo "One of the builds that is part of the experiment failed"
+elif [ $retval -eq 3 ]; then
+    echo "The build was not fully cacheable for the given task graph"
+elif [ $retval -eq 3 ]; then
+    echo "An unclassified, fatal error happened while running the experiment"
+fi
+
+exit $retval
+

.ci/bwcVersions

@@ -31,5 +31,6 @@ BWC_VERSION:
   - "8.11.4"
   - "8.12.2"
   - "8.13.4"
-  - "8.14.2"
+  - "8.14.4"
   - "8.15.0"
+  - "8.16.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,5 @@
 BWC_VERSION:
   - "7.17.23"
-  - "8.14.2"
+  - "8.14.4"
   - "8.15.0"
+  - "8.16.0"

benchmarks/build.gradle

@@ -47,8 +47,8 @@ dependencies {
   api "org.openjdk.jmh:jmh-core:$versions.jmh"
   annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
   // Dependencies of JMH
-  runtimeOnly 'net.sf.jopt-simple:jopt-simple:4.6'
-  runtimeOnly 'org.apache.commons:commons-math3:3.2'
+  runtimeOnly 'net.sf.jopt-simple:jopt-simple:5.0.4'
+  runtimeOnly 'org.apache.commons:commons-math3:3.6.1'
 }
 
 // enable the JMH's BenchmarkProcessor to generate the final benchmark classes

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -41,6 +41,7 @@
 import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator;
 import org.elasticsearch.compute.operator.topn.TopNOperator;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.BlockLoader;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
@@ -189,6 +190,11 @@ public String indexName() {
                     return "benchmark";
                 }
 
+                @Override
+                public IndexSettings indexSettings() {
+                    throw new UnsupportedOperationException();
+                }
+
                 @Override
                 public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
                     return MappedFieldType.FieldExtractPreference.NONE;

benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/TermsReduceBenchmark.java

@@ -71,7 +71,7 @@ public class TermsReduceBenchmark {
     private final SearchPhaseController controller = new SearchPhaseController((task, req) -> new AggregationReduceContext.Builder() {
         @Override
         public AggregationReduceContext forPartialReduction() {
-            return new AggregationReduceContext.ForPartial(null, null, task, builder);
+            return new AggregationReduceContext.ForPartial(null, null, task, builder, b -> {});
         }
 
         @Override

branches.json

@@ -4,6 +4,9 @@
     {
       "branch": "main"
     },
+    {
+      "branch": "8.15"
+    },
     {
       "branch": "8.14"
     },

build-tools-internal/build.gradle

@@ -274,10 +274,7 @@ dependencies {
     // ensuring brought asm version brought in by spock is up-to-date
     testImplementation buildLibs.asm
     integTestImplementation buildLibs.asm
-    integTestImplementation('org.ow2.asm:asm:9.6')
-    api("org.yaml:snakeyaml") {
-      version { strictly(versions.snakeyaml) }
-    }
+    api(buildLibs.snakeyaml)
   }
   // Forcefully downgrade the jackson platform as used in production
   api enforcedPlatform(buildLibs.jackson.platform)
@@ -314,7 +311,7 @@ dependencies {
   compileOnly buildLibs.checkstyle
   compileOnly buildLibs.reflections
 
-  implementation 'com.github.javaparser:javaparser-core:3.18.0'
+  implementation buildLibs.javaparser
 
   runtimeOnly "org.elasticsearch.gradle:reaper:$version"
   testImplementation buildLibs.checkstyle

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -26,13 +26,10 @@ develocity {
     if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') {
       publishing.onlyIf { true }
       server = 'https://gradle-enterprise.elastic.co'
-    } else {
-      publishing.onlyIf {
-        server.isPresent();
-      }
+    } else if( server.isPresent() == false) {
+      publishing.onlyIf { false }
     }
 
-
     background {
       tag OS.current().name()
       tag Architecture.current().name()

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -8,7 +8,7 @@
 
 package org.elasticsearch.gradle.internal;
 
-import com.gradle.scan.plugin.BuildScanExtension;
+import com.gradle.develocity.agent.gradle.DevelocityConfiguration;
 
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
@@ -64,7 +64,7 @@ public void apply(Project target) {
             File targetFile = target.file("build/" + buildNumber + ".tar.bz2");
             File projectDir = target.getProjectDir();
             File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/");
-            BuildScanExtension extension = target.getExtensions().getByType(BuildScanExtension.class);
+            DevelocityConfiguration extension = target.getExtensions().getByType(DevelocityConfiguration.class);
             File daemonsLogDir = new File(target.getGradle().getGradleUserHomeDir(), "daemon/" + target.getGradle().getGradleVersion());
 
             getFlowScope().always(BuildFinishedFlowAction.class, spec -> {
@@ -125,7 +125,7 @@ interface Parameters extends FlowParameters {
             ListProperty<File> getFilteredFiles();
 
             @Input
-            Property<BuildScanExtension> getBuildScan();
+            Property<DevelocityConfiguration> getBuildScan();
 
         }
 
@@ -198,7 +198,7 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
                             + System.getenv("BUILDKITE_JOB_ID")
                             + "/artifacts/"
                             + artifactUuid;
-                        parameters.getBuildScan().get().link("Artifact Upload", targetLink);
+                        parameters.getBuildScan().get().getBuildScan().link("Artifact Upload", targetLink);
                     }
                 } catch (Exception e) {
                     System.out.println("Failed to upload buildkite artifact " + e.getMessage());