Merge remote-tracking branch 'upstream/main' into rescorer_retriever

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.template.yml

@@ -10,9 +10,11 @@ steps:
               - debian-12
               - opensuse-leap-15
               - oraclelinux-8
+              - oraclelinux-9
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
+              - ubuntu-2404
               - rocky-8
               - rocky-9
               - rhel-8

.buildkite/pipelines/periodic-packaging.yml

@@ -11,9 +11,11 @@ steps:
               - debian-12
               - opensuse-leap-15
               - oraclelinux-8
+              - oraclelinux-9
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
+              - ubuntu-2404
               - rocky-8
               - rocky-9
               - rhel-8
@@ -269,8 +271,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.6 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.6
+      - label: "{{matrix.image}} / 8.15.5 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.5
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -283,7 +285,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.15.6
+          BWC_VERSION: 8.15.5
 
       - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2
@@ -301,8 +303,8 @@ steps:
         env:
           BWC_VERSION: 8.16.2
 
-      - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0
+      - label: "{{matrix.image}} / 8.17.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -315,7 +317,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.0
+          BWC_VERSION: 8.17.1
 
       - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0

.buildkite/pipelines/periodic-platform-support.yml

@@ -10,9 +10,11 @@ steps:
               - debian-12
               - opensuse-leap-15
               - oraclelinux-8
+              - oraclelinux-9
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
+              - ubuntu-2404
               - rocky-8
               - rocky-9
               - rhel-8

.buildkite/pipelines/periodic.yml

@@ -287,8 +287,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.6 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.6#bwcTest
+      - label: 8.15.5 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.5#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -297,7 +297,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.15.6
+          BWC_VERSION: 8.15.5
         retry:
           automatic:
             - exit_status: "-1"
@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.0#bwcTest
+      - label: 8.17.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.0
+          BWC_VERSION: 8.17.1
         retry:
           automatic:
             - exit_status: "-1"
@@ -448,7 +448,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -490,7 +490,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -13,9 +13,11 @@ steps:
               - debian-12
               - opensuse-leap-15
               - oraclelinux-8
+              - oraclelinux-9
               - sles-15
               - ubuntu-2004
               - ubuntu-2204
+              - ubuntu-2404
               - rocky-8
               - rocky-9
               - rhel-8

.buildkite/pipelines/pull-request/part-1-entitlements.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-1-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart1
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-1.yml

@@ -1,6 +1,8 @@
 steps:
   - label: part-1
-    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart1
+    command: |
+      .buildkite/scripts/spotless.sh # This doesn't have to be part of part-1, it was just a convenient place to put it
+      .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart1
     timeout_in_minutes: 300
     agents:
       provider: gcp

.buildkite/pipelines/pull-request/part-2-entitlements.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-2-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart2
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-3-entitlements.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-3-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart3
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-4-entitlements.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-4-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-5-entitlements.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "test-entitlements"
+steps:
+  - label: part-5-entitlements
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart5
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/precommit.yml

@@ -3,7 +3,9 @@ config:
   skip-labels: []
 steps:
   - label: precommit
-    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit
+    command: |
+      .buildkite/scripts/spotless.sh
+      .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit
     timeout_in_minutes: 300
     agents:
       provider: gcp

.buildkite/scripts/gradle-configuration-cache-validation.sh

@@ -3,16 +3,16 @@
 set -euo pipefail
 
 # This is a workaround for https://github.com/gradle/gradle/issues/28159
-.ci/scripts/run-gradle.sh --no-daemon precommit
+.ci/scripts/run-gradle.sh --no-daemon $@
 
-.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2
+.ci/scripts/run-gradle.sh --configuration-cache -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 $@
 
 # Create a temporary file
 tmpOutputFile=$(mktemp)
 trap "rm $tmpOutputFile" EXIT
 
 echo "2nd run"
-.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile
+.ci/scripts/run-gradle.sh --configuration-cache -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 $@ | tee $tmpOutputFile
 
 # Check if the command was successful
 if grep -q "Configuration cache entry reused." $tmpOutputFile; then

.buildkite/scripts/spotless.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+if [[ -z "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+  echo "Not a pull request, skipping spotless"
+  exit 0
+fi
+
+if ! git diff --exit-code; then
+  echo "Changes are present before running spotless, not running"
+  git status
+  exit 0
+fi
+
+NEW_COMMIT_MESSAGE="[CI] Auto commit changes from spotless"
+PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)"
+
+echo "--- Running spotless"
+.ci/scripts/run-gradle.sh -Dscan.tag.NESTED spotlessApply
+
+if git diff --exit-code; then
+  echo "No changes found after running spotless. Don't need to auto commit."
+  exit 0
+fi
+
+if [[ "$NEW_COMMIT_MESSAGE" == "$PREVIOUS_COMMIT_MESSAGE" ]]; then
+  echo "Changes found after running spotless"
+  echo "CI already attempted to commit these changes, but the file(s) seem to have changed again."
+  echo "Please review and fix manually."
+  exit 1
+fi
+
+git config --global user.name elasticsearchmachine
+git config --global user.email '[email protected]'
+
+gh pr checkout "${BUILDKITE_PULL_REQUEST}"
+git add -u .
+git commit -m "$NEW_COMMIT_MESSAGE"
+git push
+
+# After the git push, the new commit will trigger a new build within a few seconds and this build should get cancelled
+# So, let's just sleep to give the build time to cancel itself without an error
+# If it doesn't get cancelled for some reason, then exit with an error, because we don't want this build to be green (we just don't want it to generate an error either)
+sleep 300
+exit 1

.ci/bwcVersions

@@ -14,8 +14,8 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.6"
+  - "8.15.5"
   - "8.16.2"
-  - "8.17.0"
+  - "8.17.1"
   - "8.18.0"
   - "9.0.0"

.ci/scripts/run-gradle.sh

@@ -5,35 +5,39 @@ mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d
 
 MAX_WORKERS=4
 
-# Don't run this stuff on Windows
-if ! uname -a | grep -q MING; then
-  # drop page cache and kernel slab objects on linux
-  [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
+if [ -z "$MAX_WORKER_ENV" ]; then
+    # Don't run this stuff on Windows
+    if ! uname -a | grep -q MING; then
+      # drop page cache and kernel slab objects on linux
+      [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
 
-  if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
-    MAX_WORKERS=16
-  elif [ -f /proc/cpuinfo ]; then
-    MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
-  else
-    if [[ "$OSTYPE" == "darwin"* ]]; then
-        MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
-    else
-        echo "Unsupported OS Type: $OSTYPE"
-        exit 1
-    fi
-  fi
-  if pwd | grep -v -q ^/dev/shm ; then
-    echo "Not running on a ramdisk, reducing number of workers"
-    MAX_WORKERS=$(($MAX_WORKERS*2/3))
-  fi
+      if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
+        MAX_WORKERS=16
+      elif [ -f /proc/cpuinfo ]; then
+        MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
+      else
+        if [[ "$OSTYPE" == "darwin"* ]]; then
+            MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
+        else
+            echo "Unsupported OS Type: $OSTYPE"
+            exit 1
+        fi
+      fi
+      if pwd | grep -v -q ^/dev/shm ; then
+        echo "Not running on a ramdisk, reducing number of workers"
+        MAX_WORKERS=$(($MAX_WORKERS*2/3))
+      fi
 
-  # Export glibc version as environment variable since some BWC tests are incompatible with later versions
-  export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
-fi
+      # Export glibc version as environment variable since some BWC tests are incompatible with later versions
+      export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
+    fi
 
-# Running on 2-core machines without ramdisk can make this value be 0
-if [[ "$MAX_WORKERS" == "0" ]]; then
-  MAX_WORKERS=1
+    # Running on 2-core machines without ramdisk can make this value be 0
+    if [[ "$MAX_WORKERS" == "0" ]]; then
+      MAX_WORKERS=1
+    fi
+else
+    MAX_WORKERS=$MAX_WORKER_ENV
 fi
 
 set -e

.ci/snapshotBwcVersions

@@ -1,6 +1,5 @@
 BWC_VERSION:
-  - "8.15.6"
   - "8.16.2"
-  - "8.17.0"
+  - "8.17.1"
   - "8.18.0"
   - "9.0.0"

branches.json

@@ -13,9 +13,6 @@
     {
       "branch": "8.x"
     },
-    {
-      "branch": "8.15"
-    },
     {
       "branch": "7.17"
     }